diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2ab41456..551bacf1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,6 +50,7 @@ jobs: echo "USERORG_ENABLE=${{ vars.USERORG_SERVICE_BUILD == 'true' }}" >> $GITHUB_ENV echo "LMS_ENABLE=${{ vars.LMS_SERVICE_BUILD == 'true' }}" >> $GITHUB_ENV echo "NOTIFICATION_ENABLE=${{ vars.NOTIFICATION_SERVICE_BUILD == 'true' }}" >> $GITHUB_ENV + echo "VIEWER_ENABLE=${{ vars.VIEWER_SERVICE_BUILD == 'true' }}" >> $GITHUB_ENV # Cloud Storage Provider (CSP) — lookup from repo settings, default to azure CSP="${{ vars.CLOUD_STORAGE_PROVIDER }}" @@ -147,3 +148,23 @@ jobs: tags: ${{ env.ORG_BASE }}/lern-notification-service:${{ env.IMAGE_TAG }} build-args: | CSP=${{ env.CSP }} + + # ---------------------------------------------------------------- + # VIEWER SERVICE (standalone / distributed deployment) + # ---------------------------------------------------------------- + - name: Build Viewer Service + if: env.VIEWER_ENABLE == 'true' + env: + CSP: ${{ env.CSP }} + run: ./scripts/build-local.sh --service viewer + + - name: Push Viewer Service Docker + if: env.VIEWER_ENABLE == 'true' + uses: docker/build-push-action@v4 + with: + context: . + file: build/viewer/Dockerfile + push: true + tags: ${{ env.ORG_BASE }}/lern-viewer-service:${{ env.IMAGE_TAG }} + build-args: | + CSP=${{ env.CSP }} diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 34f0a23a..c4a96712 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -201,8 +201,54 @@ jobs: name: lern-artifacts path: modules/lern/service/**/target/** + viewer-build: + needs: build-core + runs-on: ubuntu-latest + + env: + CSP: ${{ vars.CLOUD_STORAGE_PROVIDER || 'azure' }} + + services: + redis: + image: redis:4.0.0 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Restore Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ github.run_id }} + + - name: Build and Generate Coverage Report (Viewer) + run: | + # -am builds the viewer's deps (core + lms modules); viewer-service is the standalone Play app. + mvn install -P viewer,${CSP} -pl modules/viewer/actors,modules/viewer/service -am -DskipTests -Dcheckstyle.skip + mvn verify -P viewer,${CSP} -pl modules/viewer/actors,modules/viewer/service \ + -DreuseForks=false \ + -DargLine="--add-opens java.base/sun.misc=ALL-UNNAMED -Djdk.attach.allowAttachSelf=true" + + - name: Upload Viewer Artifacts + uses: actions/upload-artifact@v4 + with: + name: viewer-artifacts + path: modules/viewer/**/target/** + sonar-analysis: - needs: [userorg-build, lms-build, notification-build, lern-build] + needs: [userorg-build, lms-build, notification-build, lern-build, viewer-build] runs-on: ubuntu-latest env: @@ -252,6 +298,13 @@ jobs: path: . merge-multiple: true + - name: Download Viewer Artifacts + uses: actions/download-artifact@v4 + with: + name: viewer-artifacts + path: . + merge-multiple: true + - name: Run Aggregated SonarQube Analysis env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.gitignore b/.gitignore index ed7e5c39..d04e7644 100644 --- a/.gitignore +++ b/.gitignore @@ -101,4 +101,4 @@ scripts/.keycloak-build/ keys/ # Claude -.claude/ \ No newline at end of file +.claude/ diff --git a/build/viewer/Dockerfile b/build/viewer/Dockerfile new file mode 100644 index 00000000..000c8d2d --- /dev/null +++ b/build/viewer/Dockerfile @@ -0,0 +1,37 @@ +# Sunbird Viewer Service Dockerfile (standalone / distributed deployment) +# Stage 1: Extraction +FROM alpine:3.20 AS builder +RUN apk update && apk add unzip +WORKDIR /app +COPY modules/viewer/service/target/viewer-service-1.0-SNAPSHOT-dist.zip . +RUN unzip viewer-service-1.0-SNAPSHOT-dist.zip + +# Stage 2: Runtime +FROM eclipse-temurin:11-jre-alpine + +# Cloud Storage Provider selection — pass --build-arg CSP= at build time. +# Defaults to azure. The Maven build must have been run with the matching profile (-P ). +# cloud_storage_auth_type defaults to OIDC (Kubernetes Workload Identity). +# Override at runtime via K8s ConfigMap or: docker run -e sunbird_cloud_storage_auth_type=ACCESS_KEY +ARG CSP=azure +ENV sunbird_cloud_service_provider=${CSP} +ENV sunbird_cloud_storage_auth_type=OIDC + +RUN apk upgrade --no-cache \ + && apk add --no-cache curl "zlib>=1.3.2-r0" \ + && adduser -u 1001 -h /home/sunbird/ -D sunbird \ + && mkdir -p /home/sunbird/ + +WORKDIR /home/sunbird/ +COPY --from=builder --chown=sunbird:sunbird /app/viewer-service-1.0-SNAPSHOT /home/sunbird/viewer-service-1.0-SNAPSHOT +COPY modules/viewer/service/conf/logback.xml /home/sunbird/viewer-service-1.0-SNAPSHOT/conf/logback.xml + +USER sunbird +EXPOSE 9000 + +CMD java -XX:+PrintFlagsFinal $JAVA_OPTIONS \ + -Dlog4j2.formatMsgNoLookups=true \ + -Dplay.server.http.idleTimeout=180s \ + -Dlogback.configurationFile=/home/sunbird/viewer-service-1.0-SNAPSHOT/conf/logback.xml \ + -cp '/home/sunbird/viewer-service-1.0-SNAPSHOT/lib/viewer-service-1.0-SNAPSHOT.jar:/home/sunbird/viewer-service-1.0-SNAPSHOT/lib/*' \ + play.core.server.ProdServerStart /home/sunbird/viewer-service-1.0-SNAPSHOT diff --git a/core/sunbird-platform-common/src/main/java/org/sunbird/keys/JsonKey.java b/core/sunbird-platform-common/src/main/java/org/sunbird/keys/JsonKey.java index 39b35767..885dfb2e 100644 --- a/core/sunbird-platform-common/src/main/java/org/sunbird/keys/JsonKey.java +++ b/core/sunbird-platform-common/src/main/java/org/sunbird/keys/JsonKey.java @@ -472,6 +472,7 @@ public final class JsonKey { public static final String CLOUD_FOLDER_CONTENT = "sunbird_cloud_content_folder"; public static final String CLOUD_STORE_BASE_PATH = "cloud_storage_base_url"; + public static final String CLOUD_STORAGE_REGION = "cloud_storage_region"; public static final String CLOUD_STORAGE_CNAME_URL= "cloud_storage_cname_url"; public static final String CLOUD_STORE_BASE_PATH_PLACEHOLDER = "cloud_store_base_path_placeholder"; public static final String TTL = "ttl"; diff --git a/core/sunbird-platform-common/src/main/java/org/sunbird/utils/CloudStorageUtil.java b/core/sunbird-platform-common/src/main/java/org/sunbird/utils/CloudStorageUtil.java index 7bc02b02..0f3a500f 100644 --- a/core/sunbird-platform-common/src/main/java/org/sunbird/utils/CloudStorageUtil.java +++ b/core/sunbird-platform-common/src/main/java/org/sunbird/utils/CloudStorageUtil.java @@ -175,6 +175,12 @@ private static IStorageService getStorageService( if (authType == StorageConfig.AuthType.ACCESS_KEY) { builder.storageSecret(storageSecret); } + // Region for providers that need it (e.g. AWS S3 outside us-east-1). No-op when unset: + // the provider SDK keeps its default (AWS -> us-east-1), so Azure/existing behaviour is unchanged. + String region = ProjectUtil.getConfigValue(JsonKey.CLOUD_STORAGE_REGION); + if (StringUtils.isNotBlank(region)) { + builder.region(region); + } StorageConfig storageConfig = builder.build(); IStorageService storageService = StorageServiceFactory.getStorageService(storageConfig); storageServiceMap.put(compositeKey, storageService); diff --git a/core/sunbird-platform-common/src/main/resources/externalresource.properties b/core/sunbird-platform-common/src/main/resources/externalresource.properties index ae0f95b0..c184eb03 100644 --- a/core/sunbird-platform-common/src/main/resources/externalresource.properties +++ b/core/sunbird-platform-common/src/main/resources/externalresource.properties @@ -71,6 +71,8 @@ sunbird_api_request_lower_case_fields=source,externalId,userName,provider,loginI # Provide corresponding service provider container(azure,aws,gcloud) sunbird_content_cloud_storage_container=sunbird-content-dev sunbird_cloud_content_folder=content +# Object-key prefix (folder) for viewer summary CSV export; blank = container root +viewer_summary_upload_path=viewer-summary sunbird_time_zone=Asia/Kolkata sunbird_health_check_enable=true sunbird_sync_read_wait_time=1500 @@ -103,6 +105,8 @@ kafka_assessment_topic= sunbird_api_mgr_base_url=https://dev.sunbirded.org/api enrollment_list_size=1000 cloud_storage_base_url=https://sunbirddev.blob.core.windows.net +# Region for providers that need it (e.g. AWS S3 outside us-east-1); blank = provider default +cloud_storage_region= cloud_store_base_path_placeholder=CLOUD_BASE_PATH #Release-5.3.0 - LR-556 content_service_mock_enabled=false @@ -178,6 +182,12 @@ sunbird_username_num_digits=4 sunbird_user_bulk_upload_size=1001 bulk_upload_org_data_size=300 sunbird_framework_read_api=/v1/framework/read +lp_meta_cache_ttl=3600 +framework_category_cache_ttl=86400 sunbird.channel.create.api.url=/channel/v3/create sunbird.channel.update.api.url=/channel/v3/update -frameworkvalidation=false \ No newline at end of file +frameworkvalidation=false +viewer_service_base_url=http://lern-service:9000 +viewer_enabled=false +# LP transport topology: monolith (in-JVM) | distributed (HTTP). +deployment_mode=monolith \ No newline at end of file diff --git a/modules/lern/service/app/controllers/viewer/ViewAggregateController.java b/modules/lern/service/app/controllers/viewer/ViewAggregateController.java new file mode 100644 index 00000000..362b8d5e --- /dev/null +++ b/modules/lern/service/app/controllers/viewer/ViewAggregateController.java @@ -0,0 +1,52 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.keys.JsonKey; +import org.sunbird.request.Request; +import org.sunbird.response.ResponseCode; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Viewer resync API (monolith wiring) — recomputes a learner's collection roll-up from + * user_content_consumption. Idempotent (recompute-from-source): backfill, drift repair, or a + * recompute after a collection is republished. Mirrors the legacy POST /v1/activity/agg, but targets + * the viewer aggregator (viewer-aggregator-actor, op "aggregate"). Runs on the userId-hashed pool, so + * a resync serialises with any live /v1/view/end for that learner — no race with real-time roll-ups. + * POST /v1/view/agg { request: { userId, collectionId|courseId, contextId|batchId } } + */ +public class ViewAggregateController extends BaseController { + + @Inject + @Named("viewer-aggregator-actor") + private ActorRef viewerAggregatorActor; + + public CompletionStage agg(Http.Request httpRequest) { + try { + Request request = createAndInitRequest("aggregate", httpRequest.body().asJson(), httpRequest); + validate(request); + return actorResponseHandler(viewerAggregatorActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + private void validate(Request request) { + String userId = (String) request.get(JsonKey.USER_ID); + Object courseId = request.get(JsonKey.COURSE_ID); + if (userId == null || userId.trim().isEmpty() + || courseId == null || courseId.toString().trim().isEmpty()) { + throw new ProjectCommonException( + ResponseCode.mandatoryParamsMissing.getErrorCode(), + "userId and courseId are mandatory", + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } +} diff --git a/modules/lern/service/app/controllers/viewer/ViewController.java b/modules/lern/service/app/controllers/viewer/ViewController.java new file mode 100644 index 00000000..f85a2917 --- /dev/null +++ b/modules/lern/service/app/controllers/viewer/ViewController.java @@ -0,0 +1,59 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.request.Request; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Viewer Service — granular view lifecycle APIs (monolith wiring). + * Dispatches to view-consumption-actor (bound in LernServiceActorStartModule). + * POST /v1/view/start -> viewStart + * POST /v1/view/update -> viewUpdate + * POST /v1/view/end -> viewEnd (synchronous recursive roll-up before responding) + */ +public class ViewController extends BaseController { + + @Inject + @Named("view-consumption-actor") + private ActorRef viewConsumptionActor; + + public CompletionStage viewStart(Http.Request httpRequest) { + return dispatch("viewStart", httpRequest); + } + + public CompletionStage viewUpdate(Http.Request httpRequest) { + return dispatch("viewUpdate", httpRequest); + } + + public CompletionStage viewEnd(Http.Request httpRequest) { + return dispatch("viewEnd", httpRequest); + } + + public CompletionStage viewRead(Http.Request httpRequest) { + return dispatch("viewRead", httpRequest); + } + + public CompletionStage assessmentSubmit(Http.Request httpRequest) { + return dispatch("viewAssess", httpRequest); + } + + public CompletionStage assessmentRead(Http.Request httpRequest) { + return dispatch("assessmentRead", httpRequest); + } + + private CompletionStage dispatch(String operation, Http.Request httpRequest) { + try { + Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + return actorResponseHandler(viewConsumptionActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } +} diff --git a/modules/lern/service/app/controllers/viewer/ViewSummaryController.java b/modules/lern/service/app/controllers/viewer/ViewSummaryController.java new file mode 100644 index 00000000..a6deeda1 --- /dev/null +++ b/modules/lern/service/app/controllers/viewer/ViewSummaryController.java @@ -0,0 +1,73 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.request.Request; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Viewer Service — read + summary APIs (monolith wiring). + * Dispatches to viewer-summary-actor (bound in LernServiceActorStartModule). + * POST /v1/summary/read + * GET /v1/summary/list/:userId + * DELETE /v1/summary/delete/:userId + */ +public class ViewSummaryController extends BaseController { + + @Inject + @Named("viewer-summary-actor") + private ActorRef viewerSummaryActor; + + public CompletionStage summaryRead(Http.Request httpRequest) { + return dispatchBody("summaryRead", httpRequest); + } + + public CompletionStage summaryList(String userId, Http.Request httpRequest) { + try { + Request request = createAndInitRequest("summaryList", httpRequest); + request.getRequest().put("userId", userId); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + public CompletionStage summaryDownload(String userId, Http.Request httpRequest) { + try { + Request request = createAndInitRequest("summaryDownload", httpRequest); + request.getRequest().put("userId", userId); + String[] fmt = httpRequest.queryString().getOrDefault("format", new String[]{"json"}); + request.getRequest().put("format", fmt.length > 0 ? fmt[0] : "json"); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + public CompletionStage summaryDelete(String userId, Http.Request httpRequest) { + try { + Request request = httpRequest.body().asJson() != null + ? createAndInitRequest("summaryDelete", httpRequest.body().asJson(), httpRequest) + : createAndInitRequest("summaryDelete", httpRequest); + request.getRequest().put("userId", userId); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + private CompletionStage dispatchBody(String operation, Http.Request httpRequest) { + try { + Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } +} diff --git a/modules/lern/service/app/modules/LernServiceActorStartModule.java b/modules/lern/service/app/modules/LernServiceActorStartModule.java index e8db0f67..8fe32c90 100644 --- a/modules/lern/service/app/modules/LernServiceActorStartModule.java +++ b/modules/lern/service/app/modules/LernServiceActorStartModule.java @@ -1,5 +1,7 @@ package modules; +import org.apache.pekko.routing.ConsistentHashingPool; +import org.apache.pekko.routing.ConsistentHashingRouter.ConsistentHashMapper; import org.apache.pekko.routing.FromConfig; import org.apache.pekko.routing.RouterConfig; import com.google.inject.AbstractModule; @@ -15,10 +17,23 @@ import org.sunbird.notification.actor.ReadNotificationActor; import org.sunbird.notification.actor.UpdateNotificationActor; import org.sunbird.observability.actor.ObservabilityReportActor; +import org.sunbird.request.Request; +import org.sunbird.viewer.actor.ViewConsumptionActor; +import org.sunbird.viewer.actor.ViewerAggregatorActor; +import org.sunbird.viewer.actor.ViewerSummaryActor; public class LernServiceActorStartModule extends AbstractModule implements PekkoGuiceSupport { private static LoggerUtil logger = new LoggerUtil(LernServiceActorStartModule.class); + // hash key for the viewer aggregator pool = userId (per-user serialization) + private static final ConsistentHashMapper viewerUserIdHashMapper = message -> { + if (message instanceof Request) { + Object uid = ((Request) message).get("userId"); + return uid != null ? uid : ""; + } + return ""; + }; + @Override protected void configure() { logger.info("LernServiceActorStartModule: Binding actors for ALL services"); @@ -53,6 +68,16 @@ protected void configure() { bindActor(ObservabilityReportActor.class, "observability-report-actor", props -> props.withRouter(config)); logger.info("Observability actors bound"); + // 5. Bind Viewer Actors (served by this monolith's /v1/view/* routes) + bindActor(ViewConsumptionActor.class, "view-consumption-actor", props -> props.withRouter(config)); + bindActor(ViewerSummaryActor.class, "viewer-summary-actor", props -> props.withRouter(config)); + // aggregator: consistent-hashing keyed on userId serialises one user's roll-ups (config router + // can't supply a hash key for plain Request messages -> would deadLetter, hence programmatic). + bindActor(ViewerAggregatorActor.class, "viewer-aggregator-actor", + props -> props.withDispatcher("pekko.actor.viewer-dispatcher") + .withRouter(new ConsistentHashingPool(8).withHashMapper(viewerUserIdHashMapper))); + logger.info("Viewer actors bound"); + logger.info("LernServiceActorStartModule: All actors bound successfully"); } } diff --git a/modules/lern/service/conf/application.conf b/modules/lern/service/conf/application.conf index bab4b64e..3304bb3b 100644 --- a/modules/lern/service/conf/application.conf +++ b/modules/lern/service/conf/application.conf @@ -131,6 +131,18 @@ pekko { } throughput = 1 } + # Viewer actors get their OWN pool: the content-consumption adapter blocks on an in-JVM ask into the + # viewer actors (monolith) — a dedicated dispatcher avoids starving the shared tracking-dispatcher. + viewer-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + throughput = 1 + } deployment { # --- USERORG ACTORS --- @@ -272,6 +284,13 @@ pekko { # --- OBSERVABILITY ACTORS --- "/observability-report-actor" { router = smallest-mailbox-pool, nr-of-instances = 4, dispatcher = rr-dispatcher } + # --- VIEWER ACTORS --- (dedicated viewer-dispatcher; see dispatcher comment above) + # viewer-aggregator-actor is NOT here: it is bound programmatically in LernServiceActorStartModule + # with a ConsistentHashingPool + hashMapper keyed on userId (config FromConfig cannot supply a + # hash key, and unhashable messages would go to deadLetters). + "/view-consumption-actor" { router = smallest-mailbox-pool, nr-of-instances = 4, dispatcher = viewer-dispatcher } + "/viewer-summary-actor" { router = smallest-mailbox-pool, nr-of-instances = 4, dispatcher = rr-dispatcher } + # --- NOTIFICATION ACTORS --- "/HealthActor" { router = smallest-mailbox-pool, nr-of-instances = 5, dispatcher = notification-dispatcher } "/HealthActor/*" { dispatcher = pekko.actor.notification-dispatcher } diff --git a/modules/lern/service/conf/routes b/modules/lern/service/conf/routes index 415877cd..06d7f9bc 100644 --- a/modules/lern/service/conf/routes +++ b/modules/lern/service/conf/routes @@ -284,3 +284,16 @@ PATCH /v1/notification/template/action/update @controllers.notification.No GET /v1/notification/template/:action @controllers.notification.NotificationTemplateController.getAction(action: String, request: play.mvc.Http.Request) + +# --- Viewer Service APIs --- +POST /v1/view/start @controllers.viewer.ViewController.viewStart(request: play.mvc.Http.Request) +POST /v1/view/update @controllers.viewer.ViewController.viewUpdate(request: play.mvc.Http.Request) +POST /v1/view/end @controllers.viewer.ViewController.viewEnd(request: play.mvc.Http.Request) +POST /v1/view/read @controllers.viewer.ViewController.viewRead(request: play.mvc.Http.Request) +POST /v1/view/agg @controllers.viewer.ViewAggregateController.agg(request: play.mvc.Http.Request) +POST /v1/assessment/submit @controllers.viewer.ViewController.assessmentSubmit(request: play.mvc.Http.Request) +POST /v1/assessment/read @controllers.viewer.ViewController.assessmentRead(request: play.mvc.Http.Request) +POST /v1/summary/read @controllers.viewer.ViewSummaryController.summaryRead(request: play.mvc.Http.Request) +GET /v1/summary/list/:userId @controllers.viewer.ViewSummaryController.summaryList(userId: String, request: play.mvc.Http.Request) +GET /v1/summary/download/:userId @controllers.viewer.ViewSummaryController.summaryDownload(userId: String, request: play.mvc.Http.Request) +DELETE /v1/summary/delete/:userId @controllers.viewer.ViewSummaryController.summaryDelete(userId: String, request: play.mvc.Http.Request) diff --git a/modules/lern/service/pom.xml b/modules/lern/service/pom.xml index 81faf8d1..adb7aac5 100644 --- a/modules/lern/service/pom.xml +++ b/modules/lern/service/pom.xml @@ -300,6 +300,15 @@ 1.0-SNAPSHOT + + + org.sunbird + viewer-actors + 1.0-SNAPSHOT + + org.sunbird diff --git a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala index b52c11f3..c398cb80 100644 --- a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala +++ b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala @@ -14,6 +14,7 @@ import org.sunbird.request.{Request, RequestContext} import org.sunbird.response.ResponseCode import org.sunbird.enrolments.BaseEnrolmentActor import org.sunbird.helper.ServiceFactory +import org.sunbird.http.HttpClientUtil import org.sunbird.kafka.KafkaClient import org.sunbird.learner.util.Util @@ -55,7 +56,14 @@ class ActivityAggregatorActor extends BaseEnrolmentActor { val contents = if (contentsRaw != null) contentsRaw.asInstanceOf[util.List[util.Map[String, AnyRef]]] else null try { - processActivityAggregates(userId, batchId, courseId, contents, requestContext) + // Backward compatibility: the frontend keeps calling /v1/activity/agg. When the viewer is + // enabled, the Viewer Service owns the processing (see delegateToViewer); otherwise legacy. + if (isViewerEnabled) { + val token = Option(request.getContext.get(JsonKey.X_AUTH_TOKEN)).map(_.asInstanceOf[String]).orNull + delegateToViewer(userId, courseId, batchId, contents, token, requestContext) + } else { + processActivityAggregates(userId, batchId, courseId, contents, requestContext) + } sender().tell(successResponse(), self) } catch { case ex: Exception => @@ -64,6 +72,58 @@ class ActivityAggregatorActor extends BaseEnrolmentActor { } } + private def isViewerEnabled: Boolean = + java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) + + private def viewerBaseUrl: String = + Option(ProjectUtil.getConfigValue("viewer_service_base_url")).filter(StringUtils.isNotBlank) + .getOrElse("http://viewer-service:9000") + + private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) + + /** + * Backward-compat adapter for /v1/activity/agg: dispatch to the viewer view/agg APIs. Contents (if any) + * -> viewStart/viewEnd (ucc write + rollup); none -> aggregate (recompute from ucc). courseId/batchId map + * to collectionId/contextId. Transport per deployment_mode: monolith -> in-JVM tell; distributed -> HTTP. + */ + private def delegateToViewer(userId: String, courseId: String, batchId: String, + contents: util.List[util.Map[String, AnyRef]], token: String, + ctx: RequestContext): Unit = { + val headers = new util.HashMap[String, String]() {{ + put("Content-Type", "application/json") + Option(token).filter(StringUtils.isNotBlank).foreach(t => put("x-authenticated-user-token", t)) + }} + // monolith: fire-and-forget to the in-JVM viewer actor (its own rollup stays async); distributed: HTTP. + def dispatch(actorName: String, api: String, operation: String, body: util.Map[String, AnyRef]): Unit = { + if (isMonolith) { + val req = new Request(); req.setRequestContext(ctx); req.setOperation(operation); req.setRequest(body) + context.actorSelection("/user/" + actorName).tell(req, org.apache.pekko.actor.ActorRef.noSender) + } else { + val envelope = new util.HashMap[String, AnyRef]() {{ put(JsonKey.REQUEST, body) }} + HttpClientUtil.post(viewerBaseUrl + api, gson.toJson(envelope), headers, ctx) + } + } + if (contents != null && !contents.isEmpty) { + contents.asScala.foreach { c => + val status = Option(c.get(JsonKey.STATUS)).map(_.asInstanceOf[Number].intValue()).getOrElse(0) + val (op, api) = if (status >= 2) ("viewEnd", "/v1/view/end") else ("viewStart", "/v1/view/start") + dispatch("view-consumption-actor", api, op, new util.HashMap[String, AnyRef]() {{ + put("contentId", c.get(JsonKey.CONTENT_ID)) + put("courseId", courseId) + put("batchId", batchId) + put(JsonKey.USER_ID, userId) + Option(c.get("progressdetails")).orElse(Option(c.get("progressDetails"))).foreach(pd => put("progressDetails", pd)) + }}) + } + } else { + dispatch("viewer-aggregator-actor", "/v1/view/agg", "aggregate", new util.HashMap[String, AnyRef]() {{ + put("courseId", courseId) + put("batchId", batchId) + put(JsonKey.USER_ID, userId) + }}) + } + } + private def processActivityAggregates( userId: String, batchId: String, diff --git a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/HierarchyRelationsUtil.scala b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/HierarchyRelationsUtil.scala index 373a82ee..117c68b1 100644 --- a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/HierarchyRelationsUtil.scala +++ b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/HierarchyRelationsUtil.scala @@ -16,7 +16,10 @@ class HierarchyRelationsUtil(cassandraOperation: CassandraOperation) { private val keyspace = Option(ProjectUtil.getConfigValue("hierarchy_store_keyspace")).getOrElse("dev_hierarchy_store") private val tableName = Option(ProjectUtil.getConfigValue("hierarchy_relations_table")).getOrElse("hierarchy_relations") - def readFromDB(key: String, requestContext: RequestContext): List[String] = { + def readFromDB(key: String, requestContext: RequestContext): List[String] = + HierarchyRelationsUtil.cached(key)(readFromDBUncached(key, requestContext)) + + private def readFromDBUncached(key: String, requestContext: RequestContext): List[String] = { logger.info(requestContext, s"HierarchyRelationsUtil.readFromDB: key: $key, keyspace: $keyspace, table: $tableName") try { val queryMap = new util.HashMap[String, AnyRef]() @@ -72,6 +75,16 @@ class HierarchyRelationsUtil(cassandraOperation: CassandraOperation) { ancestors } + /** + * Ordered trackable course ids for an LP root, from the `::trackablenodes` relation + * (emitted at publish). Order is preserved (unlock order); NOT de-duplicated. Empty if absent. + */ + def getTrackableNodes(rootId: String, requestContext: RequestContext): List[String] = { + val key = s"$rootId:$rootId:trackablenodes" + logger.info(requestContext, s"HierarchyRelationsUtil: Getting trackable nodes for rootId: $rootId") + readFromDB(key, requestContext) + } + def getRequiredLeafNodes(courseId: String, collectionId: String, requestContext: RequestContext): List[String] = { logger.info(requestContext, s"HierarchyRelationsUtil: Getting required leaf nodes (excluding optional) for courseId: $courseId, collectionId: $collectionId") val leafNodes = getLeafNodes(courseId, collectionId, requestContext) @@ -84,4 +97,26 @@ class HierarchyRelationsUtil(cassandraOperation: CassandraOperation) { object HierarchyRelationsUtil { def apply(cassandraOperation: CassandraOperation): HierarchyRelationsUtil = new HierarchyRelationsUtil(cassandraOperation) + + // JVM-wide TTL cache of relationship_key -> node_ids. The hierarchy tree changes only on collection + // republish; we accept up to TTL of staleness (structure-only, self-heals on next recompute) instead + // of detecting republish. Empty results are NOT cached, so a freshly-published collection isn't held + // stale. Set hierarchy_relations_cache_ttl=0 to disable. + // ponytail: TTL eviction; go version-keyed/event-driven only if republish-during-consumption bites. + private val ttlMillis: Long = + Option(ProjectUtil.getConfigValue("hierarchy_relations_cache_ttl")) + .filter(_.trim.nonEmpty).map(_.trim.toLong).getOrElse(300L) * 1000L + private val cache = new java.util.concurrent.ConcurrentHashMap[String, (Long, List[String])]() + + private def cached(key: String)(load: => List[String]): List[String] = { + if (ttlMillis <= 0) return load + val now = System.currentTimeMillis() + val hit = cache.get(key) + if (hit != null && hit._1 > now) hit._2 + else { + val v = load + if (v.nonEmpty) cache.put(key, (now + ttlMillis, v)) + v + } + } } diff --git a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala new file mode 100644 index 00000000..5cc72158 --- /dev/null +++ b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala @@ -0,0 +1,137 @@ +package org.sunbird.activity.util + +import com.fasterxml.jackson.databind.ObjectMapper +import org.sunbird.common.ProjectUtil +import org.sunbird.http.HttpUtil +import org.sunbird.request.RequestContext + +import java.util +import scala.collection.JavaConverters._ + +case class NodeMeta(primaryCategory: String, skills: Set[String], childNodes: List[String]) +case class LpMeta(policy: String, framework: String, categoryCode: String, nodes: Map[String, NodeMeta]) + +/** LP content metadata via a cached /v3/search + long-TTL framework->last-category-code cache. Reads the + * category code field (e.g. "skill"), never se_*Ids. Pure parsers on the companion; class does I/O + caching. */ +object LpPolicyUtil { + private val mapper = new ObjectMapper() + + private def result(json: String): util.Map[String, AnyRef] = + mapper.readValue(json, classOf[util.Map[String, AnyRef]]) + .getOrDefault("result", new util.HashMap[String, AnyRef]()).asInstanceOf[util.Map[String, AnyRef]] + + // every array-valued key under result (Content / Question / content), flattened + private def rows(res: util.Map[String, AnyRef]): List[util.Map[String, AnyRef]] = + res.asScala.values.collect { case l: util.List[_] => + l.asInstanceOf[util.List[util.Map[String, AnyRef]]].asScala }.flatten.toList + + private def strs(v: AnyRef): Set[String] = v match { + case l: util.List[_] => l.asScala.map(_.toString).toSet + case s: String if s.nonEmpty => Set(s) + case _ => Set.empty + } + + def parseFrameworkCategoryCode(json: String): Option[String] = { + val fw = result(json).getOrDefault("framework", new util.HashMap[String, AnyRef]()).asInstanceOf[util.Map[String, AnyRef]] + val cats = Option(fw.get("categories")).collect { case l: util.List[_] => + l.asInstanceOf[util.List[util.Map[String, AnyRef]]].asScala.toList }.getOrElse(Nil) + if (cats.isEmpty) None + else Some(cats.maxBy(c => Option(c.get("index")).map(_.asInstanceOf[Number].doubleValue()).getOrElse(0.0)) + .get("code").toString) + } + + def parseLpNodes(json: String, categoryCode: String): Map[String, NodeMeta] = + rows(result(json)).flatMap { r => + Option(r.get("identifier")).map(_.toString).map { id => + id -> NodeMeta( + Option(r.get("primaryCategory")).map(_.toString).getOrElse(""), + Option(if (categoryCode.isEmpty) null else r.get(categoryCode)).map(strs).getOrElse(Set.empty), + Option(r.get("childNodes")).collect { case l: util.List[_] => l.asScala.map(_.toString).toList }.getOrElse(Nil)) + } + }.toMap + + def parseField(json: String, id: String, field: String): Option[String] = + rows(result(json)).find(r => Option(r.get("identifier")).map(_.toString).contains(id)) + .flatMap(r => Option(r.get(field)).map(_.toString)) + + // ---- pure derived accessors ---- + def isAssessment(courseId: String, nodes: Map[String, NodeMeta]): Boolean = { + val self = nodes.get(courseId) + self.exists(_.primaryCategory == "Practice Question Set") || + self.toList.flatMap(_.childNodes).exists(id => nodes.get(id).exists(_.primaryCategory == "Practice Question Set")) + } + def questionSets(courseId: String, nodes: Map[String, NodeMeta]): List[String] = + nodes.get(courseId).toList.flatMap(_.childNodes).filter(id => nodes.get(id).exists(_.primaryCategory == "Practice Question Set")) + + // ---- long-TTL caches (framework changes are rare; LP structure changes only on republish) ---- + private val metaTtl: Long = + Option(ProjectUtil.getConfigValue("lp_meta_cache_ttl")).filter(_.trim.nonEmpty).map(_.trim.toLong).getOrElse(3600L) * 1000L + private val codeTtl: Long = + Option(ProjectUtil.getConfigValue("framework_category_cache_ttl")).filter(_.trim.nonEmpty).map(_.trim.toLong).getOrElse(86400L) * 1000L + private val metaCache = new java.util.concurrent.ConcurrentHashMap[String, (Long, LpMeta)]() + private val codeCache = new java.util.concurrent.ConcurrentHashMap[String, (Long, String)]() + private def cachedMeta(k: String)(load: => LpMeta): LpMeta = { + val now = System.currentTimeMillis(); val h = metaCache.get(k) + if (h != null && h._1 > now) h._2 else { val v = load; if (v.nodes.nonEmpty) metaCache.put(k, (now + metaTtl, v)); v } + } + private def cachedCode(k: String)(load: => Option[String]): Option[String] = { + val now = System.currentTimeMillis(); val h = codeCache.get(k) + if (h != null && h._1 > now) Some(h._2) else { val v = load; v.foreach(c => codeCache.put(k, (now + codeTtl, c))); v } + } + + def apply(): LpPolicyUtil = new LpPolicyUtil() +} + +class LpPolicyUtil { + import LpPolicyUtil._ + + private def searchUrl: String = ProjectUtil.getConfigValue("service_search_base_path") + "/v3/search" + + private def post(body: String): String = { + val r = HttpUtil.doPostRequest(searchUrl, body, new util.HashMap[String, String]()) + if (r != null && r.getStatusCode == 200) r.getBody else "{}" + } + + private def searchByIds(ids: List[String], fields: List[String]): String = { + if (ids.isEmpty || fields.isEmpty) return "{}" + val filters = new util.HashMap[String, AnyRef]() {{ put("status", util.Arrays.asList("Live")); put("identifier", ids.asJava) }} + val request = new util.HashMap[String, AnyRef]() {{ put("filters", filters); put("fields", fields.asJava) }} + post(mapper.writeValueAsString(new util.HashMap[String, AnyRef]() {{ put("request", request) }})) + } + + private def frameworkCategoryCode(frameworkId: String): Option[String] = + if (frameworkId == null || frameworkId.isEmpty) None + else cachedCode(frameworkId) { + val base = ProjectUtil.getConfigValue("content_service_base_url") + val api = Option(ProjectUtil.getConfigValue("sunbird_framework_read_api")).filter(_.nonEmpty).getOrElse("/v1/framework/read") + val body = HttpUtil.sendGetRequest(base + api + "/" + frameworkId, new util.HashMap[String, String]()) + parseFrameworkCategoryCode(if (body == null) "{}" else body) + } + + def lpMeta(rootId: String, ctx: RequestContext): LpMeta = cachedMeta(rootId) { + val rootJson = searchByIds(List(rootId), List("policy", "framework", "childNodes")) + val policy = parseField(rootJson, rootId, "policy").getOrElse("Strict") + val framework = parseField(rootJson, rootId, "framework").getOrElse("") + val childNodes = parseLpNodes(rootJson, "").get(rootId).map(_.childNodes).getOrElse(Nil) + val categoryCode = frameworkCategoryCode(framework).getOrElse("") + val ids = (rootId :: childNodes).distinct + val fields = if (categoryCode.isEmpty) List("primaryCategory", "childNodes") else List(categoryCode, "primaryCategory", "childNodes") + LpMeta(policy, framework, categoryCode, parseLpNodes(searchByIds(ids, fields), categoryCode)) + } + + def policyOf(meta: LpMeta): String = meta.policy match { + case p if p != null && p.equalsIgnoreCase("Adaptive") => "Adaptive" + case p if p != null && p.equalsIgnoreCase("PriorLearning") => "PriorLearning" + case _ => "Strict" + } + def isAssessmentCourse(courseId: String, meta: LpMeta): Boolean = isAssessment(courseId, meta.nodes) + def questionSetsOf(courseId: String, meta: LpMeta): List[String] = questionSets(courseId, meta.nodes) + def courseMeta(courseIds: List[String], meta: LpMeta): Map[String, (Set[String], Boolean)] = + courseIds.map(c => c -> (meta.nodes.get(c).map(_.skills).getOrElse(Set.empty), isAssessment(c, meta.nodes))).toMap + + // terms for a set of question identifiers (one /v3/search) + def skillsOfQuestions(questionIds: List[String], meta: LpMeta): Set[String] = { + if (questionIds.isEmpty || meta.categoryCode.isEmpty) return Set.empty + parseLpNodes(searchByIds(questionIds, List(meta.categoryCode)), meta.categoryCode).values.flatMap(_.skills).toSet + } +} diff --git a/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTest.scala b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTest.scala new file mode 100644 index 00000000..e041bcb0 --- /dev/null +++ b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTest.scala @@ -0,0 +1,74 @@ +package org.sunbird.activity.util + +import java.util + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.RequestContext +import org.sunbird.response.Response + +/** + * HierarchyRelationsUtil: the trackablenodes reader and the JVM-wide TTL cache on readFromDB. + * Unique relationship keys per test avoid cross-test cache bleed. + */ +class HierarchyRelationsUtilTest extends AnyFlatSpec with Matchers with MockFactory { + + private def responseWith(nodeIds: util.List[String]): Response = { + val row = new util.HashMap[String, AnyRef]() {{ put("node_ids", nodeIds) }} + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(row) + val r = new Response(); r.put("response", rows); r + } + + private def nodeList(ids: String*): util.List[String] = { + val l = new util.ArrayList[String](); ids.foreach(l.add); l + } + + // --- getTrackableNodes: reads ::trackablenodes, ordered, no dedup --- + + "getTrackableNodes" should "return the ordered trackable ids for the root" in { + val ops = mock[CassandraOperation] + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(responseWith(nodeList("CRS-A", "CRS-B", "CRS-C"))).once() + val u = HierarchyRelationsUtil(ops) + u.getTrackableNodes("trk-root-unique-1", null) shouldBe List("CRS-A", "CRS-B", "CRS-C") + } + + it should "return empty when no trackablenodes relation exists" in { + val ops = mock[CassandraOperation] + val emptyResp = new Response(); emptyResp.put("response", new util.ArrayList[util.Map[String, AnyRef]]()) + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(emptyResp).once() + val u = HierarchyRelationsUtil(ops) + u.getTrackableNodes("trk-root-unique-2", null) shouldBe empty + } + + // --- readFromDB TTL cache: repeat served from memory; empty NOT cached --- + + "getLeafNodes" should "hit the DB once and serve the repeat from cache" in { + val ops = mock[CassandraOperation] + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(responseWith(nodeList("leaf-A", "leaf-B"))).once() + val u = HierarchyRelationsUtil(ops) + val col = "cacheHit-collection-unique-1" + val first = u.getLeafNodes(col, col, null) + val second = u.getLeafNodes(col, col, null) + first should contain allOf("leaf-A", "leaf-B") + second shouldBe first + } + + "readFromDB" should "not cache empty results (freshly-published collection is re-read)" in { + val ops = mock[CassandraOperation] + val emptyResp = new Response(); emptyResp.put("response", new util.ArrayList[util.Map[String, AnyRef]]()) + val populated = responseWith(nodeList("leaf-X")) + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(emptyResp).once() + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(populated).once() + val u = HierarchyRelationsUtil(ops) + val col = "negativeCache-collection-unique-2" + u.getLeafNodes(col, col, null) shouldBe empty + u.getLeafNodes(col, col, null) should contain("leaf-X") + } +} diff --git a/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala b/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala index 15cea294..dd0880f4 100644 --- a/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala +++ b/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala @@ -18,9 +18,13 @@ class CassandraService(optionalDao: Option[CassandraOperation] = None) { private val activityTable = Option(ProjectUtil.getConfigValue("user_activity_agg_table")).getOrElse("user_activity_agg") private lazy val questionType: UserType = dao.getUDTType(keyspace, Option(ProjectUtil.getConfigValue("assessment_question_udt_type")).getOrElse("question")) + // assessment_aggregator identity columns. (user_activity_agg.context_id below is a DIFFERENT column.) + private val collectionCol: String = "course_id" + private val contextCol: String = "batch_id" + def getAssessment(aid: String, uid: String, cid: String, bid: String, contId: String, ctx: RequestContext): Option[ExistingAssessment] = { try { - val filters = buildFilters("attempt_id" -> aid, "user_id" -> uid, "course_id" -> cid, "batch_id" -> bid, "content_id" -> contId) + val filters = buildFilters("attempt_id" -> aid, "user_id" -> uid, collectionCol -> cid, contextCol -> bid, "content_id" -> contId) val fields = java.util.Arrays.asList("attempt_id", "content_id", "last_attempted_on", "created_on", "total_score", "total_max_score", "question") val records = fetchRecords(filters, fields, ctx) records.headOption.map(mapToExisting) @@ -29,7 +33,7 @@ class CassandraService(optionalDao: Option[CassandraOperation] = None) { def getUserAssessments(uid: String, cid: String, bid: String, contId: String, ctx: RequestContext): List[ExistingAssessment] = { try { - val filters = buildFilters("user_id" -> uid, "course_id" -> cid, "batch_id" -> bid, "content_id" -> contId) + val filters = buildFilters("user_id" -> uid, collectionCol -> cid, contextCol -> bid, "content_id" -> contId) val fields = java.util.Arrays.asList("content_id", "attempt_id", "last_attempted_on", "total_max_score", "total_score", "question") fetchRecords(filters, fields, ctx).map(mapToExisting) } catch { case e: Throwable => logger.error(s"List failed for $uid", e); List.empty } @@ -77,8 +81,8 @@ class CassandraService(optionalDao: Option[CassandraOperation] = None) { rec.putAll(Map( "attempt_id" -> res.attemptId, "user_id" -> res.userId, - "course_id" -> res.courseId, - "batch_id" -> res.batchId, + collectionCol -> res.courseId, + contextCol -> res.batchId, "content_id" -> res.contentId, "total_score" -> res.totalScore.asInstanceOf[AnyRef], "total_max_score" -> res.totalMaxScore.asInstanceOf[AnyRef], diff --git a/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java b/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java index acd4c7eb..3caabd9d 100644 --- a/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java +++ b/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java @@ -6,8 +6,11 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.sunbird.actor.base.BaseActor; +import org.sunbird.cassandra.CassandraOperation; +import org.sunbird.helper.ServiceFactory; import org.sunbird.common.ElasticSearchHelper; import org.sunbird.exception.ProjectCommonException; +import org.sunbird.response.ResponseCode; import org.sunbird.common.factory.EsClientFactory; import org.sunbird.common.inf.ElasticSearchService; import org.sunbird.response.Response; @@ -43,6 +46,10 @@ public class CourseBatchManagementActor extends BaseActor { private CourseBatchDao courseBatchDao = new CourseBatchDaoImpl(); + private CassandraOperation cassandraOperation = ServiceFactory.getInstance(); + // Context key carrying the top-level (root) batch id down the auto-create recursion, so every + // chained descendant batch is one level under the root (`rootBatch:courseId`), never deep-chained. + private static final String ROOT_BATCH_ID = "rootBatchId"; private UserOrgService userOrgService = UserOrgServiceImpl.getInstance(); private UserCoursesService userCoursesService = new UserCoursesService(); private ElasticSearchService esService = EsClientFactory.getInstance(); @@ -82,7 +89,11 @@ private void createCourseBatch(Request actorMessage) throws Throwable { Map request = actorMessage.getRequest(); Map targetObject; List> correlatedObject = new ArrayList<>(); - String courseBatchId = ProjectUtil.getUniqueIdFromTimestamp(actorMessage.getEnv()); + // Use a client-supplied batchId when present (e.g. chained trackable-node batch ids); else generate. + String requestedBatchId = (String) request.get(JsonKey.BATCH_ID); + String courseBatchId = StringUtils.isNotBlank(requestedBatchId) + ? requestedBatchId + : ProjectUtil.getUniqueIdFromTimestamp(actorMessage.getEnv()); Map headers = (Map) actorMessage.getContext().get(JsonKey.HEADER); String requestedBy = (String) actorMessage.getContext().get(JsonKey.REQUESTED_BY); @@ -127,6 +138,74 @@ private void createCourseBatch(Request actorMessage) throws Throwable { if (courseNotificationActive()) { batchOperationNotifier(actorMessage, courseBatch, null); } + // A trackable collection whose trackablenodes contain other trackable collections auto-creates a + // batch for each trackable child by re-triggering this same actor; every child batch is one level + // under the root batch (`rootBatch:courseId`). A collection with no trackablenodes ⇒ no-op. + // Best-effort — the batch response is already sent above. + String rootBatchId = (String) actorMessage.getContext().getOrDefault(ROOT_BATCH_ID, courseBatchId); + triggerChildBatchCreation(actorMessage, courseId, rootBatchId); + } + + /** + * Fan out child-batch creation for a trackable collection. Reads {@code trackablenodes} for + * {@code parentCourseId}; for each trackable child, re-sends a {@code createBatch} to this actor + * with a flat root-prefixed batchId ({@code rootBatchId:childCourseId}). The root batch id is + * threaded through unchanged so deeper nesting stays one level (never {@code root:parent:child}). + * The existence guard makes it idempotent, so a child already reached via the flat trackablenodes + * list is not re-created when its parent recurses. + */ + private void triggerChildBatchCreation(Request parent, String parentCourseId, String rootBatchId) { + RequestContext ctx = parent.getRequestContext(); + try { + for (String childCourseId : getTrackableNodes(parentCourseId, ctx)) { + if (StringUtils.equalsIgnoreCase(childCourseId, parentCourseId)) continue; // guard: self-reference + String childBatchId = rootBatchId + ":" + childCourseId; + if (batchExists(childCourseId, childBatchId, ctx)) continue; // idempotent: safe re-run / gap-fill + Request child = new Request(ctx); + child.setOperation("createBatch"); + child.getContext().putAll(parent.getContext()); + child.getContext().put(ROOT_BATCH_ID, rootBatchId); // keep root prefix through recursion + Map childReq = new HashMap<>(parent.getRequest()); + childReq.put(JsonKey.COURSE_ID, childCourseId); + childReq.put(JsonKey.BATCH_ID, childBatchId); + // Child batches inherit the LP root's certificate template (carried through via parent.getRequest()), + // so every trackable child issues the LP certificate on its own completion. Intentionally NOT stripped. + child.setRequest(childReq); + logger.info(ctx, "triggerChildBatchCreation: creating batch " + childBatchId + " for course " + childCourseId); + self().tell(child, ActorRef.noSender()); + } + } catch (Exception ex) { + logger.error(ctx, "triggerChildBatchCreation failed for course=" + parentCourseId + ": " + ex.getMessage(), ex); + } + } + + // Exists via readById; only invalidCourseBatchId means absent — any other PCE is rethrown so a transient + // error isn't misread as absent (which would create a duplicate batch). + private boolean batchExists(String courseId, String batchId, RequestContext ctx) { + try { + courseBatchDao.readById(courseId, batchId, ctx); + return true; + } catch (ProjectCommonException e) { + if (ResponseCode.invalidCourseBatchId.getErrorCode().equals(e.getErrorCode())) return false; + throw e; + } + } + + /** Reads the ordered trackable course ids from the `hierarchy_relations` row emitted at publish. */ + @SuppressWarnings("unchecked") + private List getTrackableNodes(String rootId, RequestContext ctx) { + String keyspace = Optional.ofNullable(ProjectUtil.getConfigValue("hierarchy_store_keyspace")) + .filter(StringUtils::isNotBlank).orElse("dev_hierarchy_store"); + String table = Optional.ofNullable(ProjectUtil.getConfigValue("hierarchy_relations_table")) + .filter(StringUtils::isNotBlank).orElse("hierarchy_relations"); + Map filters = new HashMap<>(); + filters.put("relationship_key", rootId + ":" + rootId + ":trackablenodes"); + List> rows = (List>) cassandraOperation + .getRecordsByProperties(keyspace, table, filters, ctx) + .getResult().getOrDefault(JsonKey.RESPONSE, new ArrayList<>()); + if (rows.isEmpty()) return Collections.emptyList(); + List nodeIds = (List) rows.get(0).get("node_ids"); + return nodeIds == null ? Collections.emptyList() : nodeIds; } private boolean courseNotificationActive() { diff --git a/modules/lms/course-mw/course-actors/src/test/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActorTest.java b/modules/lms/course-mw/course-actors/src/test/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActorTest.java index cc05cc26..7a07fd0b 100644 --- a/modules/lms/course-mw/course-actors/src/test/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActorTest.java +++ b/modules/lms/course-mw/course-actors/src/test/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActorTest.java @@ -3,13 +3,17 @@ import static org.powermock.api.mockito.PowerMockito.when; import org.apache.pekko.dispatch.Futures; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -138,6 +142,90 @@ public void getBatchSuccess() { Assert.assertNotNull(response); } + @Test + @PrepareForTest({ + ServiceFactory.class, + EsClientFactory.class, + UserOrgServiceImpl.class, + ContentUtil.class, InstructionEventGenerator.class, KafkaClient.class + }) + public void autoCreatesFlatChainedChildBatchesForTrackableNodes() throws Exception { + group = + MockerBuilder.getFreshMockerGroup() + .withCassandraMock(new CassandraMocker()) + .withESMock(new ESMocker()) + .withUserOrgMock(new UserOrgMocker()) + .andStaticMock(ContentUtil.class); + Map courseBatch = + CustomObjectBuilder.getCourseBatchBuilder().generateRandomFields().build().get(); + // Deterministic ids so we can assert the chained child batch ids. + courseBatch.put(JsonKey.COURSE_ID, "ROOT"); + courseBatch.put(JsonKey.BATCH_ID, "ROOT"); + + when(group.getESMockerService() + .save(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap(), Mockito.any())) + .thenReturn(Futures.successful("randomESID")); + when(ContentUtil.searchContent(Mockito.anyString(), Mockito.anyMap())) + .thenReturn(CustomObjectBuilder.getRandomCourse().get()); + + // trackablenodes read: ROOT has [C1, C2]; every other node has none (so recursion terminates). + Response trackable = new Response(); + Map row = new HashMap<>(); + row.put("node_ids", Arrays.asList("C1", "C2")); + trackable.put(JsonKey.RESPONSE, new ArrayList<>(Arrays.asList(row))); + Response emptyRel = new Response(); + emptyRel.put(JsonKey.RESPONSE, new ArrayList<>()); + when(group.getCassandraMockerService() + .getRecordsByProperties(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap(), Mockito.any())) + .thenAnswer(inv -> { + Map filters = inv.getArgument(2); + return "ROOT:ROOT:trackablenodes".equals(filters.get("relationship_key")) ? trackable : emptyRel; + }); + + // batchExists() existence check: no batch exists yet (readById throws on empty). + Response noBatch = new Response(); + noBatch.put(JsonKey.RESPONSE, new ArrayList<>()); + when(group.getCassandraMockerService() + .getRecordByIdentifier(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap(), Mockito.any(), Mockito.any())) + .thenReturn(noBatch); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(Map.class); + when(group.getCassandraMockerService() + .insertRecord(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap(), Mockito.any())) + .thenReturn(new CustomObjectBuilder.CustomObjectWrapper(true).asCassandraResponse()); + when(group.getUserOrgMockerService().getOrganisationById(Mockito.anyString())) + .thenReturn(CustomObjectBuilder.getRandomOrg().get()); + PowerMockito.mockStatic(InstructionEventGenerator.class); + PowerMockito.mockStatic(KafkaClient.class); + PowerMockito.doNothing().when(InstructionEventGenerator.class, "pushInstructionEvent", + Mockito.anyString(), Mockito.anyMap()); + PowerMockito.doNothing().when(KafkaClient.class, "send", Mockito.anyString(), Mockito.anyString()); + String orgId = ((List) courseBatch.get(JsonKey.COURSE_CREATED_FOR)).get(0); + when(group.getUserOrgMockerService().getUsersByIds(Mockito.anyList(), Mockito.anyString())) + .then((Answer>>) invocation -> + CustomObjectBuilder.getRandomUsersWithIds((List) invocation.getArguments()[0], orgId).get()); + when(group.getUserOrgMockerService().getUserById(Mockito.anyString(), Mockito.anyString())) + .then((Answer>) invocation -> + CustomObjectBuilder.getRandomUsersWithIds(Arrays.asList((String) invocation.getArguments()[0]), orgId).get().get(0)); + PowerMockito.mockStatic(ContentUtil.class); + mockCourseEnrollmentActor(); + + Request req = new Request(); + req.setOperation("createBatch"); + req.setRequest(courseBatch); + Response response = executeInTenSeconds(req, Response.class); + Assert.assertNotNull(response); + + // Root batch inserts synchronously; the two child batches are created via async self-messages. + Mockito.verify(group.getCassandraMockerService(), Mockito.timeout(8000).atLeast(3)) + .insertRecord(Mockito.anyString(), Mockito.anyString(), insertCaptor.capture(), Mockito.any()); + Set insertedValues = new HashSet<>(); + for (Map m : insertCaptor.getAllValues()) + for (Object v : m.values()) if (v instanceof String) insertedValues.add((String) v); + Assert.assertTrue("child batch ROOT:C1 not created", insertedValues.contains("ROOT:C1")); + Assert.assertTrue("child batch ROOT:C2 not created", insertedValues.contains("ROOT:C2")); + } + private void mockCourseEnrollmentActor(){ Map courseMap = new HashMap() {{ put("content", new HashMap() {{ diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala index 64665794..b9068edd 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala @@ -49,8 +49,10 @@ object AssessmentAuditRecorder { private def createRecordMap(m: util.Map[String, AnyRef], aid: String, uid: String, cid: String, ts: Long, ctx: RequestContext): util.Map[String, AnyRef] = { val rec = new util.HashMap[String, AnyRef]() - rec.put("user_id", uid); rec.put("course_id", m.get(JsonKey.COURSE_ID)) - rec.put("batch_id", m.get(JsonKey.BATCH_ID)); rec.put("content_id", cid) + rec.put("user_id", uid) + rec.put("course_id", m.get(JsonKey.COURSE_ID)) + rec.put("batch_id", m.get(JsonKey.BATCH_ID)) + rec.put("content_id", cid) rec.put("attempt_id", aid) logger.info(ctx, s"AssessmentAuditRecorder: Recording attemptId=$aid with last_attempted_on=$ts") rec.put("last_attempted_on", new java.sql.Timestamp(ts)) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala index 988a5c0e..41f239b0 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala @@ -14,6 +14,7 @@ import org.sunbird.response.ResponseCode import org.sunbird.utils.JsonUtil import org.sunbird.common.ProjectUtil import org.sunbird.helper.ServiceFactory +import org.sunbird.http.HttpClientUtil import org.sunbird.kafka.{InstructionEventGenerator, KafkaClient} import org.sunbird.learner.constants.{CourseJsonKey, InstructionEvent} import org.sunbird.learner.util.{CourseBatchUtil, Util} @@ -23,6 +24,10 @@ import java.util import java.util.{Date, TimeZone, UUID} import javax.inject.{Inject, Named} import org.apache.pekko.actor.ActorRef +import org.apache.pekko.pattern.ask +import org.apache.pekko.util.Timeout +import scala.concurrent.Await +import scala.concurrent.duration._ import scala.collection.JavaConverters._ import scala.collection.convert.ImplicitConversions._ @@ -83,24 +88,35 @@ class ContentConsumptionActor @Inject() ( val requestContext = request.getRequestContext val assessmentEvents = request.getRequest.getOrDefault(JsonKey.ASSESSMENT_EVENTS, new java.util.ArrayList[java.util.Map[String, AnyRef]]).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] val contentList = request.getRequest.getOrDefault(JsonKey.CONTENTS, new java.util.ArrayList[java.util.Map[String, AnyRef]]).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] - val finalContentList = if(CollectionUtils.isNotEmpty(assessmentEvents)) { - logger.info(requestContext, "Assessment Consumption events exist: " + assessmentEvents.size()) - val assessmentConsumptions = assessmentEvents.map(e => { - InternalContentConsumption(e.get("courseId").asInstanceOf[String], e.get("batchId").asInstanceOf[String], e.get("contentId").asInstanceOf[String]) - }).filter(cc => cc.validConsumption()).map(cc => { - var consumption: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() - consumption.put("courseId", cc.courseId) - consumption.put("batchId", cc.batchId) - consumption.put("contentId", cc.contentId) - consumption.put("status", 2.asInstanceOf[AnyRef]) - consumption - }) - if (CollectionUtils.isNotEmpty(contentList)) (contentList ++ assessmentConsumptions).asJava else assessmentConsumptions.asJava - } else contentList - logger.info(requestContext, "Final content-consumption data: " + finalContentList) - // Update consumption first and then push the assessment events if there are any. This will help us handling failures of max attempts (for assessment content). - val contentConsumptionResponse = processContents(finalContentList, requestContext, requestBy, requestedFor) - val assessmentResponse = processAssessments(assessmentEvents, requestContext, requestBy, requestedFor) + // viewer.enabled -> the Viewer Service OWNS both consumption AND assessment processing: + // regular contents -> /v1/view/start|end (ucc write + recursive rollup) + // assessment events -> /v1/assessment/submit (score + status=2 + same rollup) + // Assessments are NOT merged into the content list here (submit marks completion itself, no + // double rollup) and legacy processContents/processAssessments are skipped. API contract preserved. + val contentConsumptionResponse = + if (isViewerEnabled) delegateContentsToViewer(contentList, request, requestBy, requestedFor) + else { + // legacy path only: merge assessment events (as completed contents) into the content list. + val finalContentList = if(CollectionUtils.isNotEmpty(assessmentEvents)) { + logger.info(requestContext, "Assessment Consumption events exist: " + assessmentEvents.size()) + val assessmentConsumptions = assessmentEvents.map(e => { + InternalContentConsumption(e.get("courseId").asInstanceOf[String], e.get("batchId").asInstanceOf[String], e.get("contentId").asInstanceOf[String]) + }).filter(cc => cc.validConsumption()).map(cc => { + val consumption: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() + consumption.put("courseId", cc.courseId) + consumption.put("batchId", cc.batchId) + consumption.put("contentId", cc.contentId) + consumption.put("status", 2.asInstanceOf[AnyRef]) + consumption + }) + if (CollectionUtils.isNotEmpty(contentList)) (contentList ++ assessmentConsumptions).asJava else assessmentConsumptions.asJava + } else contentList + logger.info(requestContext, "Final content-consumption data: " + finalContentList) + processContents(finalContentList, requestContext, requestBy, requestedFor) + } + val assessmentResponse = + if (isViewerEnabled) delegateAssessmentsToViewer(assessmentEvents, request, requestBy, requestedFor) + else processAssessments(assessmentEvents, requestContext, requestBy, requestedFor) val finalResponse = assessmentResponse.getOrElse(new Response()) finalResponse.putAll(contentConsumptionResponse.getOrElse(new Response()).getResult) sender().tell(finalResponse, self) @@ -192,8 +208,7 @@ class ContentConsumptionActor @Inject() ( userContents.foreach(entry => { val userId = entry._1 if(validUserIds.contains(userId)) { - val courseId = if (entry._2.head.containsKey(JsonKey.COURSE_ID)) entry._2.head.getOrDefault(JsonKey.COURSE_ID, "").asInstanceOf[String] else entry._2.head.getOrDefault(JsonKey.COLLECTION_ID, "").asInstanceOf[String] - if(entry._2.head.containsKey(JsonKey.COLLECTION_ID)) entry._2.head.remove(JsonKey.COLLECTION_ID) + val courseId = entry._2.head.getOrDefault(JsonKey.COURSE_ID, "").asInstanceOf[String] val contentIds = entry._2.map(e => e.getOrDefault(JsonKey.CONTENT_ID, "").asInstanceOf[String]).distinct.asJava val existingContents = getContentsConsumption(userId, courseId, contentIds, batchId, requestContext).groupBy(x => x.get("contentId").asInstanceOf[String]).map(e => e._1 -> e._2.toList.head).toMap val contents:List[java.util.Map[String, AnyRef]] = entry._2.toList.map(inputContent => { @@ -429,18 +444,200 @@ class ContentConsumptionActor @Inject() ( activityAggregatorActor ! activityRequest } + // ProjectUtil.getConfigValue = env var first, then properties file. + private def isViewerEnabled: Boolean = + java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) + + // Transport for the viewer adapter: monolith -> in-JVM actor ask; distributed -> HTTP (default monolith). + private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) + private val viewerAskTimeoutMs: Long = + Option(ProjectUtil.getConfigValue("viewer_ask_timeout_ms")).filter(StringUtils.isNotBlank).map(_.trim.toLong).getOrElse(30000L) + private implicit val viewerAskTimeout: Timeout = Timeout(viewerAskTimeoutMs.millis) + private def viewerBaseUrl: String = + Option(ProjectUtil.getConfigValue("viewer_service_base_url")).filter(StringUtils.isNotBlank).getOrElse("http://viewer-service:9000") + private def viewerHeaders(token: String): java.util.Map[String, String] = + new java.util.HashMap[String, String]() {{ + put("Content-Type", "application/json") + Option(token).filter(StringUtils.isNotBlank).foreach(t => put("x-authenticated-user-token", t)) + }} + // Blocks until the viewer actor replies (rollup stays async inside the viewer); mirrors the old HTTP await. + private def viewerAsk(actorName: String, operation: String, body: java.util.Map[String, AnyRef], ctx: RequestContext): AnyRef = { + val req = new Request(); req.setRequestContext(ctx); req.setOperation(operation); req.setRequest(body) + Await.result(context.actorSelection("/user/" + actorName) ? req, viewerAskTimeoutMs.millis).asInstanceOf[AnyRef] + } + /** Write op (view/start|end, assessment/submit): monolith asks the in-JVM actor, distributed POSTs. */ + private def viewerWrite(actorName: String, httpApi: String, operation: String, + body: java.util.Map[String, AnyRef], token: String, ctx: RequestContext): Boolean = + if (isMonolith) viewerAsk(actorName, operation, body, ctx) != null + else StringUtils.isNotBlank(HttpClientUtil.post(viewerBaseUrl + httpApi, + mapper.writeValueAsString(new java.util.HashMap[String, AnyRef]() {{ put(JsonKey.REQUEST, body) }}), viewerHeaders(token), ctx)) + /** Read op (view/read): ucc rows. Monolith returns native Cassandra types; distributed parses JSON + coerces. */ + private def viewerRead(actorName: String, httpApi: String, operation: String, + body: java.util.Map[String, AnyRef], token: String, ctx: RequestContext): java.util.List[java.util.Map[String, AnyRef]] = { + val empty = new java.util.ArrayList[java.util.Map[String, AnyRef]]() + if (isMonolith) viewerAsk(actorName, operation, body, ctx) match { + case r: Response => r.getResult.getOrDefault(JsonKey.RESPONSE, empty).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + case _ => empty + } else { + val responseStr = HttpClientUtil.post(viewerBaseUrl + httpApi, + mapper.writeValueAsString(new java.util.HashMap[String, AnyRef]() {{ put(JsonKey.REQUEST, body) }}), viewerHeaders(token), ctx) + if (StringUtils.isBlank(responseStr)) empty + else { + val respNode = mapper.readTree(responseStr).path("result").path("response") + val rows = mapper.convertValue(respNode, classOf[java.util.List[java.util.Map[String, AnyRef]]]) + rows.asScala.foreach(coerceUccRowTypes) + rows + } + } + } + + /** + * Backward-compat adapter for content/state/update (content items): extract the fields and dispatch + * each to the viewer view API (status >= 2 -> viewEnd, else viewStart). The viewer owns the ucc write + * + async rollup. Idempotent monotonic upserts, so retries are safe. Response: contentId -> SUCCESS/FAILED. + */ + private def delegateContentsToViewer(contentList: java.util.List[java.util.Map[String, AnyRef]], + originalRequest: Request, + requestedBy: String, requestedFor: String): Option[Response] = { + if (CollectionUtils.isEmpty(contentList)) return None + val ctx = originalRequest.getRequestContext + val userId = if (StringUtils.isNotBlank(requestedFor)) requestedFor else requestedBy + val token = originalRequest.getContext.get(JsonKey.X_AUTH_TOKEN).asInstanceOf[String] + val responseMessage = new java.util.HashMap[String, AnyRef]() + contentList.asScala.foreach(c => { + val contentId = c.get(JsonKey.CONTENT_ID).asInstanceOf[String] + try { + val status = c.getOrDefault(JsonKey.STATUS, 0.asInstanceOf[AnyRef]).asInstanceOf[Number].intValue() + val courseId = c.get(JsonKey.COURSE_ID).asInstanceOf[String] + val (op, api) = if (status >= 2) ("viewEnd", "/v1/view/end") else ("viewStart", "/v1/view/start") + val body = new java.util.HashMap[String, AnyRef]() {{ + put("contentId", contentId) + put("courseId", courseId) + put("batchId", c.get(JsonKey.BATCH_ID)) + put(JsonKey.USER_ID, userId) + Option(c.get("progressdetails")).orElse(Option(c.get("progressDetails"))).foreach(pd => put("progressDetails", pd)) + }} + responseMessage.put(contentId, if (viewerWrite("view-consumption-actor", api, op, body, token, ctx)) JsonKey.SUCCESS else "FAILED") + } catch { + case ex: Exception => + logger.error(ctx, s"delegateContentsToViewer failed for contentId=$contentId: ${ex.getMessage}", ex) + responseMessage.put(contentId, "FAILED") + } + }) + val response = new Response(); response.putAll(responseMessage); Option(response) + } + + /** + * Backward-compat adapter for content/state/update (assessment events): dispatch each to the viewer + * assessment/submit (score + status=2 + rollup). Applies the SAME batch-validity guard as legacy + * processAssessments: unknown batch -> BATCH_NOT_EXISTS, completed batch (status != 1) -> + * NOT_A_ON_GOING_BATCH; only ongoing batches are dispatched. Response: batchId -> SUCCESS/FAILED + * (+ the two error keys when applicable). + */ + private def delegateAssessmentsToViewer(assessmentEvents: java.util.List[java.util.Map[String, AnyRef]], + originalRequest: Request, + requestedBy: String, requestedFor: String): Option[Response] = { + if (CollectionUtils.isEmpty(assessmentEvents)) return None + val ctx = originalRequest.getRequestContext + val userId = if (StringUtils.isNotBlank(requestedFor)) requestedFor else requestedBy + val token = originalRequest.getContext.get(JsonKey.X_AUTH_TOKEN).asInstanceOf[String] + val byBatch: Map[String, List[java.util.Map[String, AnyRef]]] = assessmentEvents.asScala + .filter(e => StringUtils.isNotBlank(e.getOrDefault(JsonKey.BATCH_ID, "").asInstanceOf[String])).toList + .groupBy(_.get(JsonKey.BATCH_ID).asInstanceOf[String]) + val batchIds = byBatch.keySet.toList.asJava + val batches: Map[String, List[java.util.Map[String, AnyRef]]] = + getBatches(ctx, new java.util.ArrayList[String](batchIds), null).toList.groupBy(_.get(JsonKey.BATCH_ID).asInstanceOf[String]) + val invalidBatchIds = byBatch.keySet.diff(batches.keySet).toList.asJava + batches.values.foreach(bl => bl.foreach(b => CourseBatchUtil.enrichBatchStatusFromDates(b))) + val completedBatchIds = batches.filter(b => 1 != b._2.head.get(JsonKey.STATUS).asInstanceOf[Integer]).keys.toList.asJava + val responseMessage = new java.util.HashMap[String, AnyRef]() + byBatch.foreach { case (batchId, events) => + if (!invalidBatchIds.contains(batchId) && !completedBatchIds.contains(batchId)) { + events.foreach(a => { + try { + val courseId = a.get(JsonKey.COURSE_ID).asInstanceOf[String] + val evs = a.getOrDefault(JsonKey.ASSESSMENT_EVENTS_KEY, new java.util.ArrayList[java.util.Map[String, AnyRef]]()) + val body = new java.util.HashMap[String, AnyRef]() {{ + put("contentId", a.get(JsonKey.CONTENT_ID)) + put("courseId", courseId) + put("batchId", batchId) + put(JsonKey.USER_ID, userId) + put(JsonKey.ASSESSMENT_EVENTS, evs) + }} + responseMessage.put(batchId, if (viewerWrite("view-consumption-actor", "/v1/assessment/submit", "viewAssess", body, token, ctx)) JsonKey.SUCCESS else "FAILED") + } catch { + case ex: Exception => + logger.error(ctx, s"delegateAssessmentsToViewer failed for batchId=$batchId: ${ex.getMessage}", ex) + responseMessage.put(batchId, "FAILED") + } + }) + } + } + if (CollectionUtils.isNotEmpty(completedBatchIds)) responseMessage.put("NOT_A_ON_GOING_BATCH", completedBatchIds) + if (CollectionUtils.isNotEmpty(invalidBatchIds)) responseMessage.put("BATCH_NOT_EXISTS", invalidBatchIds) + val response = new Response(); response.putAll(responseMessage); Option(response) + } + + /** + * Backward-compat adapter for content/state/read: fetch the ucc rows from the viewer (viewRead) instead + * of a direct Cassandra read. Returns the same row shape getContentsConsumption produces (monolith: + * native Cassandra types; distributed: coerced back from JSON), so the caller's post-processing is unchanged. + */ + private def readContentsFromViewer(userId: String, courseId: String, batchId: String, + contentIds: java.util.List[String], originalRequest: Request): java.util.List[java.util.Map[String, AnyRef]] = { + val ctx = originalRequest.getRequestContext + val token = originalRequest.getContext.get(JsonKey.X_AUTH_TOKEN).asInstanceOf[String] + val body = new java.util.HashMap[String, AnyRef]() {{ + put(JsonKey.USER_ID, userId) + put("courseId", courseId) + put("batchId", batchId) + if (CollectionUtils.isNotEmpty(contentIds)) put("contentId", contentIds) + }} + try viewerRead("view-consumption-actor", "/v1/view/read", "viewRead", body, token, ctx) + catch { + case ex: Exception => + logger.error(ctx, s"readContentsFromViewer failed for userId=$userId courseId=$courseId: ${ex.getMessage}", ex) + new java.util.ArrayList[java.util.Map[String, AnyRef]]() + } + } + + // ucc column -> native type, so JSON-parsed viewer rows match a direct Cassandra read exactly. + private val uccTimestampCols = Set("last_access_time", "last_completed_time", "last_updated_time", "datetime") + private val uccIntCols = Set("status", "progress", "completedcount", "viewcount") + + private def coerceUccRowTypes(row: java.util.Map[String, AnyRef]): Unit = { + uccTimestampCols.foreach { c => val d = toDate(row.get(c)); if (d != null) row.put(c, d) } + uccIntCols.foreach { c => row.get(c) match { case n: Number => row.put(c, Integer.valueOf(n.intValue())); case _ => } } + row.get("completionpercentage") match { case n: Number => row.put("completionpercentage", java.lang.Float.valueOf(n.floatValue())); case _ => } + } + + /** Reconstruct a java.util.Date from the viewer's JSON form (epoch-millis number, numeric string, or ISO-8601). */ + private def toDate(v: AnyRef): java.util.Date = v match { + case null => null + case d: java.util.Date => d + case n: Number => new java.util.Date(n.longValue()) + case s: String if StringUtils.isNotBlank(s) => + try new java.util.Date(s.toLong) + catch { case _: Throwable => try java.util.Date.from(java.time.Instant.parse(s)) catch { case _: Throwable => null } } + case _ => null + } + def getConsumption(request: Request): Unit = { val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] val batchId = request.get(JsonKey.BATCH_ID).asInstanceOf[String] val courseId = request.get(JsonKey.COURSE_ID).asInstanceOf[String] val contentIds = request.getRequest.getOrDefault(JsonKey.CONTENT_IDS, new java.util.ArrayList[String]()).asInstanceOf[java.util.List[String]] val fields = request.getRequest.getOrDefault(JsonKey.FIELDS, new java.util.ArrayList[String](){{ add(JsonKey.PROGRESS) }}).asInstanceOf[java.util.List[String]] - val contentsConsumed = getContentsConsumption(userId, courseId, contentIds, batchId, request.getRequestContext) + // viewer.enabled -> Content-State-Read is served by the Viewer Service (/v1/view/read), so raw + // ucc reads can be archived away from this API. Same rows, so the existing post-processing + // (field filtering, json parse, date format, optional assessment score) is reused unchanged. + val contentsConsumed = + if (isViewerEnabled) readContentsFromViewer(userId, courseId, batchId, contentIds, request) + else getContentsConsumption(userId, courseId, contentIds, batchId, request.getRequestContext) val response = new Response if(CollectionUtils.isNotEmpty(contentsConsumed)) { val filteredContents = contentsConsumed.map(m => { ProjectUtil.removeUnwantedFields(m, JsonKey.DATE_TIME, JsonKey.USER_ID, JsonKey.ADDED_BY, JsonKey.LAST_UPDATED_TIME, JsonKey.OLD_LAST_ACCESS_TIME, JsonKey.OLD_LAST_UPDATED_TIME, JsonKey.OLD_LAST_COMPLETED_TIME) - m.put(JsonKey.COLLECTION_ID, m.getOrDefault(JsonKey.COURSE_ID, "")) jsonFields.foreach(field => if(m.get(field) != null) m.put(field, mapper.readTree(m.get(field).asInstanceOf[String])) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index cb25608d..3c49a665 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -7,6 +7,7 @@ import org.apache.commons.lang3.StringUtils import org.sunbird.cache.util.RedisCacheUtil import org.sunbird.common.CassandraUtil import org.sunbird.exception.ProjectCommonException +import org.sunbird.http.HttpClientUtil import org.sunbird.response.Response import org.sunbird.common.ProjectUtil import org.sunbird.common.ProjectUtil.EnrolmentType @@ -90,6 +91,7 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c validateEnrolment(batchData, enrolmentData, true) val data: java.util.Map[String, AnyRef] = createUserEnrolmentMap(userId, courseId, batchId, enrolmentData, request.getContext.getOrDefault(JsonKey.REQUEST_ID, "").asInstanceOf[String]) upsertEnrollment(userId, courseId, batchId, data, (null == enrolmentData), request.getRequestContext) + logger.info(request.getRequestContext, s"enrol: enrolled | user=$userId course=$courseId batch=$batchId new=${null == enrolmentData}") if (isCacheEnabled) { logger.info(request.getRequestContext, "CourseEnrolmentActor :: enroll :: Deleting redis for key " + getCacheKey(userId)) cacheUtil.delete(getCacheKey(userId)) @@ -97,6 +99,31 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c sender().tell(successResponse(), self) generateTelemetryAudit(userId, courseId, batchId, data, "enrol", JsonKey.CREATE, request.getContext) notifyUser(userId, batchData, JsonKey.ADD) + fireLpBootstrap(userId, courseId, batchId, request) + } + + /** + * Open the first LP course right after a USER enrol (design Step 3): fire the viewer rollup once so + * advanceLp runs at enrol time instead of only on first consumption. Skips system-lp child enrols + * (already driven by their rollup) and no-ops when the viewer is off. Fire-and-forget; monolith -> + * in-JVM to the aggregator, distributed -> HTTP /v1/view/agg. Mirrors ContentConsumptionActor's transport. + */ + private def fireLpBootstrap(userId: String, courseId: String, batchId: String, request: Request): Unit = { + val requestId = Option(request.getContext.get(JsonKey.REQUEST_ID)).map(_.toString).getOrElse("") + if ("system".equals(requestId)) return + if (!java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled"))) return + try { + if (!"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode"))) { + val agg = new Request(); agg.setRequestContext(request.getRequestContext); agg.setOperation("aggregate") + agg.put(JsonKey.USER_ID, userId); agg.put("courseId", courseId); agg.put("batchId", batchId) + context.actorSelection("/user/viewer-aggregator-actor").tell(agg, ActorRef.noSender) + } else { + val base = Option(ProjectUtil.getConfigValue("viewer_service_base_url")).filter(StringUtils.isNotBlank).getOrElse("http://viewer-service:9000") + val headers = new util.HashMap[String, String]() {{ put("Content-Type", "application/json") }} + HttpClientUtil.post(base + "/v1/view/agg", s"""{"request":{"userId":"$userId","courseId":"$courseId","batchId":"$batchId"}}""", headers, request.getRequestContext) + } + logger.info(request.getRequestContext, s"enrol: LP bootstrap fired | user=$userId course=$courseId batch=$batchId") + } catch { case ex: Exception => logger.error(request.getRequestContext, s"enrol: LP bootstrap failed: ${ex.getMessage}", ex) } } @@ -164,7 +191,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c enrolment.put(JsonKey.LEAF_NODE_COUNT, courseContent.get(JsonKey.LEAF_NODE_COUNT)) enrolment.put(JsonKey.COURSE_LOGO_URL, courseContent.get(JsonKey.APP_ICON)) enrolment.put(JsonKey.CONTENT_ID, enrolment.get(JsonKey.COURSE_ID)) - enrolment.put(JsonKey.COLLECTION_ID, enrolment.get(JsonKey.COURSE_ID)) enrolment.put(JsonKey.CONTENT, courseContent) enrolment }).toList.asJava diff --git a/modules/lms/course-mw/enrolment-actor/src/test/scala/org/sunbird/enrolments/CourseConsumptionActorTest.scala b/modules/lms/course-mw/enrolment-actor/src/test/scala/org/sunbird/enrolments/CourseConsumptionActorTest.scala index 6fc82a29..51e3e76d 100644 --- a/modules/lms/course-mw/enrolment-actor/src/test/scala/org/sunbird/enrolments/CourseConsumptionActorTest.scala +++ b/modules/lms/course-mw/enrolment-actor/src/test/scala/org/sunbird/enrolments/CourseConsumptionActorTest.scala @@ -295,7 +295,7 @@ class CourseConsumptionActorTest extends FlatSpec with Matchers with MockFactory ((keyspace: _root_.scala.Predef.String, table: _root_.scala.Predef.String, filters: _root_.java.util.Map[_root_.scala.Predef.String, AnyRef], fields: _root_.java.util.List[_root_.scala.Predef.String], requestContext: RequestContext) => cassandraOperation.getRecords(keyspace, table, filters, fields, requestContext)).expects(*, *, *, *, *).returns(response) val result = callActor(getStateReadRequestWithProgressField(), Props(new ContentConsumptionActor(mockActivityAggregatorActor, mockAssessmentAggregatorActor).setCassandraOperation(cassandraOperation, false))) - result.getResult().get("response").toString.shouldEqual("[{progressDetails={key1=val1, key2=val2}, contentId=do_456, batchId=0123, courseId=do_123, collectionId=do_123, progressdetails={}}]") + result.getResult().get("response").toString.shouldEqual("[{progressDetails={key1=val1, key2=val2}, contentId=do_456, batchId=0123, courseId=do_123, progressdetails={}}]") assert(null != result) } diff --git a/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java b/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java index cc025784..467c6322 100644 --- a/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java +++ b/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java @@ -39,8 +39,6 @@ public CompletionStage createBatch(Http.Request httpRequest) { httpRequest.body().asJson(), (request) -> { Request req = (Request) request; - String courseId = req.getRequest().containsKey(JsonKey.COURSE_ID) ? JsonKey.COURSE_ID : JsonKey.COLLECTION_ID; - req.getRequest().put(JsonKey.COURSE_ID, req.getRequest().get(courseId)); new CourseBatchRequestValidator().validateCreateCourseBatchRequest(req); return null; }, @@ -69,8 +67,6 @@ public CompletionStage updateBatch(Http.Request httpRequest) { httpRequest.body().asJson(), (request) -> { Request req = (Request) request; - String courseId = req.getRequest().containsKey(JsonKey.COURSE_ID) ? JsonKey.COURSE_ID : JsonKey.COLLECTION_ID; - req.getRequest().put(JsonKey.COURSE_ID, req.getRequest().get(courseId)); new CourseBatchRequestValidator().validateUpdateCourseBatchRequest(req); return null; }, @@ -98,7 +94,8 @@ public CompletionStage search(Http.Request httpRequest) { if (reqObj.getRequest().containsKey(JsonKey.FILTERS) && reqObj.getRequest().get(JsonKey.FILTERS) != null && reqObj.getRequest().get(JsonKey.FILTERS) instanceof Map) { - ((Map) (reqObj.getRequest().get(JsonKey.FILTERS))).put(JsonKey.OBJECT_TYPE, esObjectType); + Map f = (Map) reqObj.getRequest().get(JsonKey.FILTERS); + f.put(JsonKey.OBJECT_TYPE, esObjectType); } else { Map filtermap = new HashMap<>(); Map dataMap = new HashMap<>(); diff --git a/modules/viewer/actors/pom.xml b/modules/viewer/actors/pom.xml new file mode 100644 index 00000000..4c57e555 --- /dev/null +++ b/modules/viewer/actors/pom.xml @@ -0,0 +1,175 @@ + + + + + org.sunbird + viewer + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + viewer-actors + Viewer Actors + Viewer actors: granular view lifecycle + recursive collection tracking + + + 11 + 11 + 2.13 + 2.13.12 + 1.0.3 + UTF-8 + + + + + + org.scala-lang + scala-library + ${scala.maj.version} + + + org.apache.pekko + pekko-actor_${scala.version} + ${pekko.version} + + + + + org.sunbird + course-actors-common + 1.0-SNAPSHOT + + + org.sunbird + sunbird-actor-utils + 1.0-SNAPSHOT + + + org.sunbird + enrolment-actor + 1.0-SNAPSHOT + + + org.sunbird + sunbird-cache-utils + + + + + + org.sunbird + activity-aggregator + 1.0-SNAPSHOT + + + org.sunbird + assessment-aggregator + 1.0-SNAPSHOT + + + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.commons + commons-lang3 + 3.12.0 + + + org.apache.commons + commons-collections4 + 4.4 + + + + + org.scalatest + scalatest_${scala.version} + 3.2.15 + test + + + org.scalamock + scalamock_${scala.version} + 5.2.0 + test + + + org.apache.pekko + pekko-testkit_${scala.version} + ${pekko.version} + test + + + + + src/main/scala + src/test/scala + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + ${scala.maj.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + + org.scalatest + scalatest-maven-plugin + 2.0.0 + + + test + test + + test + + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar + + + + + + + diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala new file mode 100644 index 00000000..e784f6d7 --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala @@ -0,0 +1,296 @@ +package org.sunbird.viewer.actor + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.commons.collections4.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.pekko.actor.ActorRef +import org.sunbird.assessment.models._ +import org.sunbird.assessment.service.{AssessmentService, CassandraService, ContentService} +import org.sunbird.assessment.util.AssessmentParser +import org.sunbird.common.ProjectUtil +import org.sunbird.enrolments.BaseEnrolmentActor +import org.sunbird.helper.ServiceFactory +import org.sunbird.keys.JsonKey +import org.sunbird.learner.util.Util +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.response.Response + +import java.util +import javax.inject.{Inject, Named} +import scala.collection.JavaConverters._ + +/** + * Granular view lifecycle writes to user_content_consumption (ucc). + * + * Write model = read-modify-upsert with MONOTONIC merge (mirrors ContentConsumptionActor; + * NOT Paxos LWT). Race-free by monotonicity + Cassandra per-cell LWW + per-userId serialization: + * viewStart -> INSERT only if absent (status 1); if present, no-op. + * viewUpdate -> merge only if row exists; revisits (already status 2) ignored. + * viewEnd -> status 2 + completed time; then async aggregation (fire-and-forget tell). + * + * ucc PK (viewer schema §2): (userid, courseid, batchid, contentid). + * No collection context -> courseid = batchid = contentid. + */ +class ViewConsumptionActor @Inject() ( + @Named("viewer-aggregator-actor") viewerAggregatorActor: ActorRef +) extends BaseEnrolmentActor { + + private val mapper = new ObjectMapper + private var cassandraOperation = ServiceFactory.getInstance + private val consumptionDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_CONTENT_DB) + private val CONSUMPTION_TABLE = "user_content_consumption" + private val enrolmentDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_COURSE_DB) + + /** + * Stamp the enrolment's last-content-access on every view op (mirrors standard content-consumption): + * user_enrolments.lastcontentaccesstime/lastreadcontentid/lastreadcontentstatus. Keyed by the ucc + * primary key (userid, courseid, batchid). This is what summary/list surfaces as access time. + */ + private def touchEnrolmentAccess(key: util.HashMap[String, AnyRef], status: Int, ctx: RequestContext): Unit = { + val selectMap = new util.HashMap[String, AnyRef]() {{ + put("userid", key.get("userid")); put("courseid", key.get("courseid")); put("batchid", key.get("batchid")) + }} + // Only stamp an EXISTING enrolment. updateRecordV2's ifExists is a no-op (plain UPDATE upserts in + // Cassandra), so without this guard a no-context/unenrolled view would fabricate a phantom enrolment row. + val existing = cassandraOperation.getRecordByIdentifier(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (existing.isEmpty) { logger.info(ctx, s"view: access skip(no-enrolment) | user=${key.get("userid")} course=${key.get("courseid")} batch=${key.get("batchid")}"); return } + val updateMap = new util.HashMap[String, AnyRef]() {{ + put("lastcontentaccesstime", new java.util.Date()) + put("lastreadcontentid", key.get("contentid")) + put("lastreadcontentstatus", Integer.valueOf(status)) + }} + cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) + logger.info(ctx, s"view: access stamped | user=${key.get("userid")} course=${key.get("courseid")} content=${key.get("contentid")} status=$status") + } + + // Assessment scoring reuses the assessment-aggregator services in-process (same math + persistence + // the legacy AssessmentAggregatorActor uses). ContentService is only touched if metadata validation + // is invoked — the viewer submit path does not call it, so no content-search network hop. + private lazy val assessmentService = new AssessmentService(new ContentService()) + // built on the same injected CassandraOperation, so setCassandraOperation() also controls the + // assessment persistence in tests (and keeps a single Cassandra handle in production). + private lazy val assessmentCassandra = new CassandraService(Some(cassandraOperation)) + + override def onReceive(request: Request): Unit = { + request.getOperation match { + case "viewStart" => viewStart(request) + case "viewUpdate" => viewUpdate(request) + case "viewEnd" => viewEnd(request) + case "viewRead" => viewRead(request) + case "viewAssess" => viewAssess(request) + case "assessmentRead" => assessmentRead(request) + case _ => onReceiveUnsupportedOperation(request.getOperation) + } + } + + /** + * /v1/assessment/submit (api.view.assess). Scores the attempt (reuse AssessmentService + + * CassandraService — identical to legacy AssessmentAggregatorActor), then treats the assessment + * like a completed content: mark ucc status=2 and run the same sync rollup as viewEnd, so the + * assessment leaf counts toward collection completion. Score aggregates (score:cid/max_score:cid) + * land in user_activity_agg via putAll append; the rollup's completion agg uses different keys → + * they coexist. Request: userId, courseId?, batchId?, contentId, assessments[] (assess events). + */ + private def viewAssess(request: Request): Unit = { + val ctx = request.getRequestContext + val key = viewKey(request) + val userId = key.get("userid").asInstanceOf[String] + val courseId = key.get("courseid").asInstanceOf[String] + val batchId = key.get("batchid").asInstanceOf[String] + val contentId = key.get("contentid").asInstanceOf[String] + + val eventsRaw = Option(request.get(JsonKey.ASSESSMENT_EVENTS)).orElse(Option(request.get(JsonKey.EVENTS))) + .map(_.asInstanceOf[util.List[util.Map[String, AnyRef]]]).getOrElse(new util.ArrayList[util.Map[String, AnyRef]]()) + val events: List[AssessmentEvent] = eventsRaw.asScala.map(AssessmentParser.mapToEvent).toList + + if (events.nonEmpty) { + val ts = Option(request.get("assessmentTs")).orElse(Option(request.get("assessmentTimestamp"))) + .map(_.asInstanceOf[Number].longValue()).getOrElse(System.currentTimeMillis()) + val attemptId = Option(request.get(JsonKey.ATTEMPT_ID)).map(_.toString).filter(StringUtils.isNotBlank) + .getOrElse(java.util.UUID.randomUUID().toString) // no client attemptId -> a fresh attempt (avoids hashCode collisions overwriting a prior attempt) + val unique = assessmentService.getUniqueQuestions(events) + val metrics = assessmentService.computeScoreMetrics(unique) + val result = AssessmentResult(attemptId, userId, courseId, batchId, contentId, + metrics.totalScore, metrics.totalMaxScore, metrics.grandTotal, metrics.questions, System.currentTimeMillis(), ts) + assessmentCassandra.saveAssessment(result, ctx) + // best-score across all attempts -> user_activity_agg (reuse legacy aggregation) + val stored = assessmentCassandra.getUserAssessments(userId, courseId, batchId, contentId, ctx) + val agg = assessmentService.computeUserAggregates(userId, courseId, batchId, stored) + assessmentCassandra.updateUserActivity(userId, courseId, batchId, agg, ctx) + } else { + logger.warn(ctx, s"viewAssess: no assessment events for userId=$userId contentId=$contentId; marking complete only", null) + } + + // Assessment content is complete on submit (score-independent, mirrors legacy status=2) -> ucc + rollup. + val row = new util.HashMap[String, AnyRef](key) + row.put("status", Integer.valueOf(2)) + row.put("last_completed_time", ProjectUtil.getTimeStamp) + row.put("last_updated_time", ProjectUtil.getTimeStamp) + cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + touchEnrolmentAccess(key, 2, ctx) + triggerAggregation(request, ctx) + + val out = new Response(); out.put(contentId, JsonKey.SUCCESS); sender().tell(out, self) + } + + /** + * /v1/assessment/read (api.assessment.read). Best score / max score per content from + * assessment_aggregator (reuse getUserAssessments). Request: userId, contentId[] , courseId?, batchId?. + */ + private def assessmentRead(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] + val courseId = ViewerRequestKeys.courseId(request).orNull + val batchId = ViewerRequestKeys.batchId(request).orNull + val contentIds: List[String] = request.get("contentId") match { + case l: util.List[_] => l.asScala.map(_.asInstanceOf[String]).toList + case s: String if StringUtils.isNotBlank(s) => List(s) + case _ => List.empty + } + val contents = new util.ArrayList[util.Map[String, AnyRef]]() + contentIds.foreach { cid => + val stored = assessmentCassandra.getUserAssessments(userId, courseId, batchId, cid, ctx) + if (stored.nonEmpty) { + val best = stored.maxBy(_.totalScore) + val m = new util.HashMap[String, AnyRef]() + m.put("identifier", cid) + m.put("score", best.totalScore.asInstanceOf[AnyRef]) + m.put("max_score", best.totalMaxScore.asInstanceOf[AnyRef]) + contents.add(m) + } + } + val out = new Response() + out.put(JsonKey.USER_ID, userId) + out.put("courseId", courseId) + out.put("batchId", batchId) + out.put("contents", contents) + sender().tell(out, self) + } + + /** Raw ucc rows for a user's content(s) under a collection. context=all -> ignore batchid. */ + private def viewRead(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] + val allContexts = "all".equalsIgnoreCase(request.get("context").asInstanceOf[String]) + val filters = new util.HashMap[String, AnyRef]() + filters.put("userid", userId) + ViewerRequestKeys.courseId(request).foreach(c => filters.put("courseid", c)) + if (!allContexts) ViewerRequestKeys.batchId(request).foreach(c => filters.put("batchid", c)) + val contentIds = request.get("contentId") match { + case l: util.List[_] => l.asScala.map(_.asInstanceOf[String]).asJava + case s: String if StringUtils.isNotBlank(s) => util.Arrays.asList(s) + case _ => null + } + if (contentIds != null && !contentIds.isEmpty) filters.put("contentid", contentIds) + val response = cassandraOperation.getRecords(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + val rows = response.getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + val out = new Response(); out.put(JsonKey.RESPONSE, rows); sender().tell(out, self) + } + + private def viewStart(request: Request): Unit = { + val ctx = request.getRequestContext + val key = viewKey(request) + val existing = readRow(key, ctx) + if (existing == null) { + val row = new util.HashMap[String, AnyRef](key) + row.put("status", Integer.valueOf(1)) + row.put("last_access_time", ProjectUtil.getTimeStamp) + row.put("last_updated_time", ProjectUtil.getTimeStamp) + cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + logger.info(ctx, s"view: start inserted | user=${key.get("userid")} course=${key.get("courseid")} batch=${key.get("batchid")} content=${key.get("contentid")}") + } else logger.info(ctx, s"view: start noop(exists) | user=${key.get("userid")} content=${key.get("contentid")}") + touchEnrolmentAccess(key, 1, ctx) + sender().tell(successResponse(), self) + } + + private def viewUpdate(request: Request): Unit = { + val ctx = request.getRequestContext + val key = viewKey(request) + val existing = readRow(key, ctx) + if (existing != null && statusOf(existing) < 2) { + val row = new util.HashMap[String, AnyRef](key) + // status is monotonic; an update never downgrades and never completes (that's viewEnd) + row.put("status", Integer.valueOf(math.max(1, statusOf(existing)))) + Option(request.get("progressDetails")).foreach(pd => row.put("progressdetails", mapper.writeValueAsString(pd))) + row.put("last_access_time", ProjectUtil.getTimeStamp) + row.put("last_updated_time", ProjectUtil.getTimeStamp) + cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + logger.info(ctx, s"view: update merged | user=${key.get("userid")} content=${key.get("contentid")}") + } else logger.info(ctx, s"view: update skip(absent-or-completed) | user=${key.get("userid")} content=${key.get("contentid")}") + touchEnrolmentAccess(key, math.max(1, if (existing != null) statusOf(existing) else 1), ctx) + sender().tell(successResponse(), self) + } + + private def viewEnd(request: Request): Unit = { + val ctx = request.getRequestContext + val key = viewKey(request) + val row = new util.HashMap[String, AnyRef](key) + row.put("status", Integer.valueOf(2)) + Option(request.get("progressDetails")).foreach(pd => row.put("progressdetails", mapper.writeValueAsString(pd))) + row.put("last_completed_time", ProjectUtil.getTimeStamp) + row.put("last_updated_time", ProjectUtil.getTimeStamp) + cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + logger.info(ctx, s"view: end completed | user=${key.get("userid")} course=${key.get("courseid")} batch=${key.get("batchid")} content=${key.get("contentid")}") + touchEnrolmentAccess(key, 2, ctx) + // Async rollup: fire-and-forget tell to the aggregator; respond immediately (does not wait). + triggerAggregation(request, ctx) + sender().tell(successResponse(), self) + } + + /** Fire-and-forget tell to ViewerAggregatorActor; the rollup runs async — the response does not wait. */ + private def triggerAggregation(request: Request, ctx: RequestContext): Unit = { + val key = viewKey(request) + val aggRequest = new Request() + aggRequest.setOperation("aggregate") + aggRequest.setRequestContext(ctx) + aggRequest.put(JsonKey.USER_ID, key.get("userid")) + aggRequest.put("courseId", key.get("courseid")) + aggRequest.put("batchId", key.get("batchid")) + // Async, fire-and-forget: the rollup + LP progression run in the background on the aggregator + // (per-user serialized). The hot path does not wait for it — the change from before is ask -> tell. + logger.info(ctx, s"view: rollup triggered | user=${key.get("userid")} course=${key.get("courseid")} batch=${key.get("batchid")}") + viewerAggregatorActor.tell(aggRequest, ActorRef.noSender) + } + + /** + * Build the ucc primary key (live column names userid, courseid, batchid, contentid). + * Backward-compatible request keys: courseId (else legacy courseId), batchId (else legacy + * batchId). No collection ctx -> courseid = batchid = contentId. + */ + private def viewKey(request: Request): util.HashMap[String, AnyRef] = { + // explicit userId (internal delegation) else requestedFor/requestedBy (from token on direct API calls) + val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]).filter(StringUtils.isNotBlank) + .orElse(Option(request.get(JsonKey.REQUESTED_FOR).asInstanceOf[String]).filter(StringUtils.isNotBlank)) + .getOrElse(request.get(JsonKey.REQUESTED_BY).asInstanceOf[String]) + val contentId = ViewerRequestKeys.contentId(request) + val courseId = ViewerRequestKeys.courseId(request).getOrElse(contentId) + val batchId = ViewerRequestKeys.batchId(request).getOrElse(contentId) + val key = new util.HashMap[String, AnyRef]() + key.put("userid", userId) + key.put("courseid", courseId) + key.put("batchid", batchId) + key.put("contentid", contentId) + key + } + + private def readRow(key: util.HashMap[String, AnyRef], ctx: RequestContext): util.Map[String, AnyRef] = { + val filters = new util.HashMap[String, AnyRef](key) + val response = cassandraOperation.getRecords(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + val rows = response.getResult + .getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) rows.get(0) else null + } + + private def statusOf(row: util.Map[String, AnyRef]): Int = + Option(row.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) + + // for tests + def setCassandraOperation(ops: org.sunbird.cassandra.CassandraOperation): ViewConsumptionActor = { + cassandraOperation = ops + this + } +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala new file mode 100644 index 00000000..a2dba3b2 --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -0,0 +1,348 @@ +package org.sunbird.viewer.actor + +import org.apache.commons.collections4.CollectionUtils +import org.sunbird.activity.domain.{ContentStatus, UserContentConsumption, UserEnrolmentAgg} +import org.sunbird.activity.util.{ActivityAggregateUtil, CertificateUtil, HierarchyRelationsUtil} +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.common.ProjectUtil +import org.sunbird.enrolments.BaseEnrolmentActor +import org.sunbird.helper.ServiceFactory +import org.sunbird.keys.JsonKey +import org.sunbird.learner.util.Util +import org.sunbird.request.{Request, RequestContext} + +import java.util +import scala.collection.JavaConverters._ + +/** + * Sync, in-request recursive rollup for the viewer module. Invoked from viewEnd (per-userId serialized). + * + * REUSE: all aggregation math is ActivityAggregateUtil (same calls ActivityAggregatorActor uses). + * The util treats courseId as the activity_id slot and batchId as the context slot — it does not + * care about the names. Viewer deltas vs ActivityAggregatorActor: + * - optionality is PER-USER: `optional_nodes` from user_enrolments (NOT hierarchy getOptionalNodes). + * required = collectionLeafNodes.diff(userOpt) at leaf AND every ancestor level. + * - recompute from DB state each call -> idempotent, safe under per-user serialization. + * + * ASSUMPTIONS to verify against live schema (v2 snake_case): + * - user_content_consumption PK (user_id, collection_id, context_id, content_id); status per content. + * - user_enrolments keyed (userid, courseid, batchid) with optional_nodes set. + * - user_activity_agg is the aggregate target (activity_id = collection do-id). + */ +class ViewerAggregatorActor extends BaseEnrolmentActor { + + private var cassandraOperation: CassandraOperation = ServiceFactory.getInstance + private var hierarchyRelationsUtil: HierarchyRelationsUtil = HierarchyRelationsUtil(cassandraOperation) + private var certificateUtil: CertificateUtil = CertificateUtil() + private val activityAggUtil = new ActivityAggregateUtil() + private val lpPolicyUtil: org.sunbird.activity.util.LpPolicyUtil = org.sunbird.activity.util.LpPolicyUtil() + private val assessmentService = new org.sunbird.assessment.service.CassandraService(Some(cassandraOperation)) + + private val enrolmentDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_COURSE_DB) + private val activityAggDBInfo = Util.dbInfoMap.get(JsonKey.GROUP_ACTIVITY_DB) + private val consumptionDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_CONTENT_DB) + private val courseBatchDBInfo = Util.dbInfoMap.get(JsonKey.COURSE_BATCH_DB) + private val CONSUMPTION_TABLE = "user_content_consumption" + // Course-level certificates on course completion (the LP cert is always issued by the engine). + // Default true; set course_certificate_enabled=false to suppress course certs only. + private val courseCertEnabled: Boolean = + ViewerAggregatorActor.parseCourseCertEnabled(ProjectUtil.getConfigValue("course_certificate_enabled")) + // LP progression extracted to a focused, injectable engine (SRP); transport behind a dispatcher (OCP). + // lazy so `context` is set by the time they initialize. + private lazy val enrolDispatcher: org.sunbird.viewer.engine.EnrolDispatcher = org.sunbird.viewer.engine.EnrolDispatcher(context) + private lazy val lpEngine = new org.sunbird.viewer.engine.LpProgressionEngine( + cassandraOperation, enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, lpPolicyUtil, assessmentService, enrolDispatcher, certificateUtil) + + override def onReceive(request: Request): Unit = { + request.getOperation match { + case "aggregate" => + // Fire-and-forget rollup: log failures (caller already acked); /v1/view/agg force-sync is the repair path. + try aggregate(request) + catch { case ex: Exception => logger.error(request.getRequestContext, s"ViewerAggregatorActor.aggregate failed: ${ex.getMessage}", ex) } + sender().tell(successResponse(), self) + case _ => onReceiveUnsupportedOperation(request.getOperation) + } + } + + private def aggregate(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] + // accept courseId (else legacy courseId) / batchId (else legacy batchId) + val courseId = ViewerRequestKeys.courseId(request).orNull + val batchId = ViewerRequestKeys.batchId(request).orNull + if (userId == null || courseId == null) { + logger.warn(ctx, s"ViewerAggregatorActor: missing userId/courseId, skipping", null) + return + } + + // trackablenodes non-empty => this root is a Learning Path (structural detection; §Step 2/5). + val trackable = hierarchyRelationsUtil.getTrackableNodes(courseId, ctx) + logger.info(ctx, s"viewer.rollup: start | user=$userId course=$courseId batch=$batchId " + + (if (trackable.nonEmpty) s"LP detected n=${trackable.size}" else "not-an-LP")) + + // 1. Read this user's consumption for the (root) collection+context from viewer ucc, build status map + val rows = readConsumption(userId, courseId, batchId, ctx) + if (CollectionUtils.isEmpty(rows)) { + // No consumption yet. For an LP, still advance (bootstrap: open the first required course). + if (trackable.nonEmpty) { logger.info(ctx, s"viewer.rollup: no-consumption -> LP bootstrap | user=$userId course=$courseId"); advanceLp(userId, courseId, batchId, trackable, ctx) } + else logger.info(ctx, s"viewer.rollup: no-consumption skip | user=$userId course=$courseId") + return + } + val contentStatusMap: Map[String, ContentStatus] = activityAggUtil.getContentStatusFromContents(rows) + val uc = UserContentConsumption(userId, batchId, courseId, contentStatusMap) + + // 2. Per-learner optional COURSES from user_enrolments.optional_nodes (LP policy; course-level). + val perLearnerOptionalCourses: List[String] = readOptionalNodes(userId, courseId, batchId, ctx) + + // 3. Root leaves + the tree's nodes (via ancestors) — needed before computing effectiveOptional. + val leafNodes = hierarchyRelationsUtil.getLeafNodes(courseId, courseId, ctx) + if (leafNodes.isEmpty) { + logger.warn(ctx, s"ViewerAggregatorActor: no leafNodes for courseId=$courseId; is hierarchy_relations published?", null) + return + } + val ancestors: Map[String, List[String]] = uc.contents.map { case (contentId, content) => + (contentId, hierarchyRelationsUtil.getAncestors(courseId, content.contentId, ctx)) + }.toMap + val childCollections = ancestors.values.flatten.filter(_ != courseId).toList.distinct + + // effectiveOptional LEAVES (§5.1) = author-marked hierarchy `optionalnodes` (content-level, all nodes) + // ∪ leaves of per-learner optional courses (course-level, expanded to leaves so the leaf-vs-leaf diff works). + val treeNodes = courseId :: childCollections + val hierarchyOptionalLeaves = treeNodes.flatMap(n => hierarchyRelationsUtil.getOptionalNodes(courseId, n, ctx)).distinct + val optionalCourseLeaves = perLearnerOptionalCourses.flatMap(c => hierarchyRelationsUtil.getLeafNodes(courseId, c, ctx)).distinct + val effectiveOptional: List[String] = (hierarchyOptionalLeaves ++ optionalCourseLeaves).distinct + + // 4. Aggregates: root + every ancestor node; required per node = its leafNodes − effectiveOptional. + val courseAgg = activityAggUtil.computeCourseActivityAgg(uc, leafNodes, effectiveOptional, ctx) + val collectionsWithLeafNodes: Map[String, List[String]] = childCollections.map { col => + (col, hierarchyRelationsUtil.getLeafNodes(courseId, col, ctx).diff(effectiveOptional)) + }.toMap + val moduleAggs = activityAggUtil.computeModuleActivityAgg(uc, courseId, ancestors, collectionsWithLeafNodes, ctx) + + val allAggs: List[UserEnrolmentAgg] = courseAgg.toList ++ moduleAggs + + // 5. Write user_activity_agg (frozen content_status + agg) for root + every node + writeActivityAggregates(allAggs, ctx) + logger.info(ctx, s"viewer.rollup: nodes rolled-up n=${allAggs.size} | user=$userId course=$courseId batch=$batchId") + + // 6. Per-node progress: nodeId -> (completedCount, requiredLeaves) for root + every trackable ancestor. + val nodeProgress = scala.collection.mutable.LinkedHashMap[String, (Int, List[String])]() + courseAgg.foreach(a => nodeProgress(courseId) = (completedCountOf(a), leafNodes.diff(effectiveOptional))) + moduleAggs.foreach(a => nodeProgress(a.activityAgg.activity_id) = + (completedCountOf(a), collectionsWithLeafNodes.getOrElse(a.activityAgg.activity_id, Nil))) + + // 7. Update user_enrolments status for EVERY enrolled node in this tree (approach #1: key off the + // child enrolment rows that already exist; root included). Cert fires once, on transition to complete. + // Returns the node ids that are COMPLETE (status == 2) after this pass. + val completedNow = writeAllNodeEnrolments(userId, courseId, batchId, nodeProgress.toMap, contentStatusMap, ctx) + + // 8. LP progression: for the LP root, advance. For a chained child, bridge to the LP root when the child + // course is COMPLETE this pass (not on a partial view) — the completion is the trigger. Idempotent: + // a re-submit of an already-complete course still re-advances the LP. + if (trackable.nonEmpty) advanceLp(userId, courseId, batchId, trackable, ctx) + else if (completedNow.contains(courseId)) bridgeToRoot(userId, courseId, batchId, ctx) + } + + /** A child course just completed -> if it belongs to an LP, re-fire the LP-root aggregate so it re-advances. */ + private def bridgeToRoot(userId: String, courseId: String, courseBatchId: String, ctx: RequestContext): Unit = + parentLpOf(courseId, courseBatchId, ctx).foreach { case (lpId, lpBatch) => + val req = new Request(); req.setRequestContext(ctx); req.setOperation("aggregate") + req.put(JsonKey.USER_ID, userId); req.put("courseId", lpId); req.put("batchId", lpBatch) + self.tell(req, org.apache.pekko.actor.ActorRef.noSender) + logger.info(ctx, s"viewer.rollup: child->LP bridge | user=$userId course=$courseId childBatch=$courseBatchId lp=$lpId lpBatch=$lpBatch") + } + + /** + * Parent LP of a just-completed course, or None for a standalone course. Resolves from the authoritative + * structural map (course_batch): strip the ":" prefix to get the LP batch, read course_batch by batchid, + * and pick the row whose courseid differs from the completed course (that's the LP). Requires the + * course_batch(batchid) secondary index (see migrations) so this is a keyed read, not a scan — same + * pattern as user_enrolments_by_batch. + * ponytail: leans on the ":" child-batch convention (rootBatch:childId) to recover the LP batch id. + * Ceiling: assumes no standalone batch id contains ":". Upgrade: stamp parent_collection_id/ + * parent_context_id on the course_batch/enrolment row at creation and read those directly (migration-time). + */ + private def parentLpOf(courseId: String, courseBatchId: String, ctx: RequestContext): Option[(String, String)] = + ViewerAggregatorActor.resolveParentLp(courseId, courseBatchId, lpBatch => courseBatchRowsByBatchId(lpBatch, ctx)) + + /** course_batch rows for a batch id — keyed read via the course_batch(batchid) secondary index. */ + private def courseBatchRowsByBatchId(batchId: String, ctx: RequestContext): util.List[util.Map[String, AnyRef]] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("batchid", batchId) }} + cassandraOperation.getRecords(courseBatchDBInfo.getKeySpace, courseBatchDBInfo.getTableName, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + } + + private def completedCountOf(a: UserEnrolmentAgg): Int = + a.activityAgg.aggregates.getOrElse("completedCount", 0.0).toInt + + // LP progression: read the status snapshot + derive ancestorsOf here, then delegate to LpProgressionEngine. + private def advanceLp(userId: String, rootId: String, batchId: String, + trackable: List[String], ctx: RequestContext): Unit = { + // One read of this user's enrolments, reused for every completion/enrolment check below. Safe because + // advanceLp runs AFTER writeAllNodeEnrolments has committed this pass's statuses, so the snapshot is + // current; collapses the LP's former O(courses) single-row status reads into a single query. + val status = enrolStatusSnapshot(userId, ctx) + // Course ancestors aren't published (only leaf ancestors are), so derive a course's chain from one of + // its leaves: leaf ancestors = [..course, level, root] (root LAST) -> levelOf = last non-root = the level. + val ancestorsOf = (course: String) => + hierarchyRelationsUtil.getLeafNodes(rootId, course, ctx).headOption + .map(leaf => hierarchyRelationsUtil.getAncestors(rootId, leaf, ctx)) + .getOrElse(List.empty[String]) + lpEngine.advance(userId, rootId, batchId, trackable, status, ancestorsOf, ctx) + } + + /** + * All of this user's enrolments as (courseId, batchId) -> status, in one read. Replaces the LP's former + * per-course status queries. Result rows are camelCase (createResponse); rows without a status map to 0 + * (enrol always sets one) so presence-of-key still answers "is enrolled". + */ + private def enrolStatusSnapshot(userId: String, ctx: RequestContext): Map[(String, String), Int] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} + val rows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + rows.asScala.flatMap { r => + for { + c <- Option(r.get("courseId")).map(_.toString) + b <- Option(r.get("batchId")).map(_.toString) + } yield (c, b) -> Option(r.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) + }.toMap + } + + private def writeActivityAggregates(aggs: List[UserEnrolmentAgg], ctx: RequestContext): Unit = { + val aggQueries = aggs.map(a => activityAggUtil.createActivityAggUpdateMap(a.activityAgg)).asJava + if (!aggQueries.isEmpty) + cassandraOperation.batchUpdateWithPutAll(activityAggDBInfo.getKeySpace, activityAggDBInfo.getTableName, aggQueries, ctx) + } + + /** + * Approach #1: update user_enrolments status for every node in this tree that has an enrolment row. + * Matches each row on courseid ∈ tree AND this LP's batchid (§4: standalone enrolments untouched). + * Cert fires once, only on the transition to complete (status != 2 -> 2). + */ + private def writeAllNodeEnrolments(userId: String, rootId: String, batchId: String, + nodeProgress: Map[String, (Int, List[String])], + contentStatusMap: Map[String, ContentStatus], ctx: RequestContext): Set[String] = { + val completedNow = scala.collection.mutable.Set[String]() + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} + val enrolRows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + enrolRows.asScala.foreach { row => + // createResponse maps columns to camelCase field names (courseid->courseId, batchid->batchId + // via cassandratablecolumn.properties), so result rows are always camelCase. + val nodeId = Option(row.get("courseId")).map(_.toString).orNull + val nodeCtx = Option(row.get("batchId")).map(_.toString).orNull + // This LP only (root=batchId, child=batchId:childId); a standalone enrolment's batchid differs (§4). + val expectedCtx = if (nodeId == rootId) batchId else batchId + ":" + nodeId + nodeProgress.get(nodeId).filter(_ => nodeCtx == expectedCtx).foreach { case (completedCount, requiredLeaves) => + val required = requiredLeaves.size + val status = activityAggUtil.getCompletionStatus(completedCount, required) + val pct = activityAggUtil.getCompletionPercentage(completedCount, required) + val currentStatus = Option(row.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) + val nodeContentStatus: Map[String, AnyRef] = + requiredLeaves.flatMap(l => contentStatusMap.get(l).map(cs => l -> Integer.valueOf(cs.status).asInstanceOf[AnyRef])).toMap + // updateRecordV2 replaces the whole contentstatus column — merge into the row's existing map so a + // root-keyed rollup that only sees some leaves doesn't clobber the rest (see mergeContentStatus). + val mergedContentStatus = ViewerAggregatorActor.mergeContentStatus(row.get("contentStatus"), nodeContentStatus) + val selectMap = new util.HashMap[String, AnyRef]() {{ + put("userid", userId); put("courseid", nodeId); put("batchid", nodeCtx) + }} + val updateMap = new util.HashMap[String, AnyRef]() {{ + put("progress", Integer.valueOf(completedCount)) + put("status", Integer.valueOf(status)) + put("completionpercentage", Integer.valueOf(pct)) + put("contentstatus", mergedContentStatus) + if (status == 2 && currentStatus != 2) put("completedon", new java.util.Date()) + }} + cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) + // Complete after this pass -> eligible to bridge to the LP. Gate on COMPLETENESS (status==2), not + // the status!=2->2 transition: a course that was already complete (re-submit, or completed before + // the bridge existed) must still re-advance the LP. advanceLp is idempotent, so re-firing is safe; + // a partial pass (status 1) never bridges. + if (status == 2) completedNow += nodeId + // Cert fires once, only on the transition to complete (avoid re-issuing on a re-submit). + if (status == 2 && currentStatus != 2) { + if (courseCertEnabled) { + logger.info(ctx, s"viewer.rollup: node completed -> cert | user=$userId course=$nodeId batch=$nodeCtx") + certificateUtil.publishCertificateIssueEvent(userId, nodeId, nodeCtx, ctx) + } else logger.info(ctx, s"viewer.rollup: node completed, course cert suppressed (course_certificate_enabled=false) | user=$userId course=$nodeId batch=$nodeCtx") + } + } + } + completedNow.toSet + } + + /** + * Read this user's ucc rows for the collection, scoped to the context. + * (userid, courseid, batchid) is a clustering-prefix slice on PK + * (userid, courseid, batchid, contentid) -> efficient, no scan. + * batchId omitted only when absent (no-context viewer), falling back to collection-wide read. + */ + private def readConsumption(userId: String, courseId: String, batchId: String, ctx: RequestContext): util.List[util.Map[String, AnyRef]] = { + val filters = new util.HashMap[String, AnyRef]() {{ + put("userid", userId) + put("courseid", courseId) + if (batchId != null) put("batchid", batchId) + }} + val response = cassandraOperation.getRecords(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + response.getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + } + + /** Per-user optional_nodes from user_enrolments (empty for strict policy). */ + private def readOptionalNodes(userId: String, courseId: String, batchId: String, ctx: RequestContext): List[String] = { + val filters = new util.HashMap[String, AnyRef]() {{ + put("userid", userId) + put("courseid", courseId) + put("batchid", batchId) + }} + val response = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + val rows = response.getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) { + Option(rows.get(0).get("optional_nodes")) + .map(_.asInstanceOf[util.Collection[String]].asScala.toList) + .getOrElse(List()) + } else List() + } + + // for tests + def configure(ops: CassandraOperation, hru: HierarchyRelationsUtil, cu: CertificateUtil): ViewerAggregatorActor = { + cassandraOperation = ops; hierarchyRelationsUtil = hru; certificateUtil = cu; this + } +} + +object ViewerAggregatorActor { + // Merge freshly computed per-leaf statuses into the enrolment row's existing contentstatus map instead of + // replacing it: updateRecordV2 overwrites the whole column, so a root-keyed rollup that only sees some + // leaves must not wipe the rest (C2). `existing` may be null; fresh values win on key conflicts. + private[actor] def mergeContentStatus(existing: AnyRef, fresh: Map[String, AnyRef]): java.util.Map[String, AnyRef] = { + val merged = new java.util.HashMap[String, AnyRef]() + Option(existing).foreach(m => merged.putAll(m.asInstanceOf[java.util.Map[String, AnyRef]])) + fresh.foreach { case (k, v) => merged.put(k, v) } + merged + } + + // Parent LP of a completed course: strip the ":" child-batch prefix to get the LP batch, read course_batch + // by that batch id, and pick the row whose courseid differs from the completed course (a course may hold + // its own records under the same batch string; only a DIFFERENT courseid is the parent LP). None for a + // standalone (colon-free) batch — fetchByBatchId is not invoked in that case. + private[actor] def resolveParentLp(courseId: String, courseBatchId: String, + fetchByBatchId: String => java.util.List[java.util.Map[String, AnyRef]]): Option[(String, String)] = { + if (courseBatchId == null || !courseBatchId.contains(":")) None + else { + val lpBatch = courseBatchId.substring(0, courseBatchId.indexOf(":")) + fetchByBatchId(lpBatch).asScala + .flatMap(r => Option(r.get("courseId")).map(_.toString)) + .find(_ != courseId).map(lpId => (lpId, lpBatch)) + } + } + + // Course-cert toggle: default true; only the literal "false" disables it (the LP cert is unaffected). + private[actor] def parseCourseCertEnabled(cfg: String): Boolean = !"false".equalsIgnoreCase(Option(cfg).getOrElse("")) +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala new file mode 100644 index 00000000..e73fc6b2 --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala @@ -0,0 +1,19 @@ +package org.sunbird.viewer.actor + +import org.apache.commons.lang3.StringUtils +import org.sunbird.request.Request + +/** + * Canonical viewer request keys — the viewer contract is courseId / batchId / contentId only. + * Legacy courseId/batchId resolution is the caller's job (the content-consumption adapter maps them + * before dispatching), so no fallback lives here. Null/blank-safe extraction in one place. + */ +object ViewerRequestKeys { + + private def value(request: Request, key: String): Option[String] = + Option(request.get(key)).collect { case s: String if StringUtils.isNotBlank(s) => s } + + def courseId(request: Request): Option[String] = value(request, "courseId") + def batchId(request: Request): Option[String] = value(request, "batchId") + def contentId(request: Request): String = value(request, "contentId").orNull +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala new file mode 100644 index 00000000..bcfc8b65 --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala @@ -0,0 +1,167 @@ +package org.sunbird.viewer.actor + +import org.apache.commons.collections4.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.sunbird.common.ProjectUtil +import org.sunbird.enrolments.BaseEnrolmentActor +import org.sunbird.helper.ServiceFactory +import org.sunbird.keys.JsonKey +import org.sunbird.learner.util.Util +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.response.Response +import org.sunbird.utils.CloudStorageUtil + +import java.util +import scala.collection.JavaConverters._ + +/** + * Summary APIs for the viewer module — enrolment-level (distinct from the per-content view actor). + * Reads/deletes user_enrolments only; never writes consumption. Raw ucc reads (viewRead) live on + * ViewConsumptionActor. + * + * summaryRead -> per-enrolment progress/status/contentstatus for one collection + * summaryList -> all enrolment summaries for a user + * summaryDelete -> delete enrolment rows (all, or one collection+context) + */ +class ViewerSummaryActor extends BaseEnrolmentActor { + + private var cassandraOperation = ServiceFactory.getInstance + private val enrolmentDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_COURSE_DB) + + override def onReceive(request: Request): Unit = { + request.getOperation match { + case "summaryRead" => summaryRead(request) + case "summaryList" => summaryList(request) + case "summaryDownload" => summaryDownload(request) + case "summaryDelete" => summaryDelete(request) + case _ => onReceiveUnsupportedOperation(request.getOperation) + } + } + + /** Per-enrolment progress/status from user_enrolments (identified by courseId). */ + private def summaryRead(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] + val courseId = ViewerRequestKeys.courseId(request).orNull + val batchId = ViewerRequestKeys.batchId(request).orNull + + // Identified by courseId (courseid). user_enrolments carries progress/status/completionpercentage + // + per-content contentstatus for the enrolment — no activity_type needed. + val enrolFilters = new util.HashMap[String, AnyRef]() + enrolFilters.put("userid", userId) + if (StringUtils.isNotBlank(courseId)) enrolFilters.put("courseid", courseId) + if (StringUtils.isNotBlank(batchId)) enrolFilters.put("batchid", batchId) + val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, enrolFilters, ctx) + logger.info(ctx, s"summary: read | user=$userId course=$courseId rows=${enrolments.size}") + val response = new Response + response.put(JsonKey.RESPONSE, enrolments) + sender().tell(response, self) + } + + /** All enrolment summaries for a user (partition by user_id). */ + private def summaryList(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]) + .getOrElse(request.get("userId").asInstanceOf[String]) + val filters = new util.HashMap[String, AnyRef]() + filters.put("userid", userId) + val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters, ctx) + logger.info(ctx, s"summary: list | user=$userId rows=${enrolments.size}") + val response = new Response + response.put(JsonKey.RESPONSE, enrolments) + sender().tell(response, self) + } + + /** + * Download a user's enrolment summaries. format=json (default) returns the rows under "response"; + * format=csv uploads the CSV to cloud storage (generic CloudStorageUtil) and returns its "url". + */ + private def summaryDownload(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]) + .getOrElse(request.get("userId").asInstanceOf[String]) + val format = Option(request.get("format").asInstanceOf[String]).map(_.toLowerCase).getOrElse("json") + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} + val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters, ctx) + val response = new Response + response.put("format", format) + if (format == "csv") response.put("url", uploadSummaryCsv(userId, toCsv(enrolments))) + else response.put(JsonKey.RESPONSE, enrolments) + logger.info(ctx, s"summary: download | user=$userId format=$format rows=${enrolments.size}") + sender().tell(response, self) + } + + /** + * Write the summary CSV to a temp file and upload it to cloud storage; return the object URL. + * Provider-agnostic via CloudStorageUtil (StorageServiceFactory). All config-driven, nothing hardcoded: + * sunbird_cloud_service_provider (provider), sunbird_content_cloud_storage_container (existing container), + * viewer_summary_upload_path (object-key prefix; blank = container root). Object = /_viewer_summary.csv + */ + private def uploadSummaryCsv(userId: String, csv: String): String = { + val storageType = ProjectUtil.getConfigValue("sunbird_cloud_service_provider") + val container = ProjectUtil.getConfigValue("sunbird_content_cloud_storage_container") + val prefix = Option(ProjectUtil.getConfigValue("viewer_summary_upload_path")).getOrElse("").trim.stripSuffix("/") + val objectKey = (if (StringUtils.isNotBlank(prefix)) prefix + "/" else "") + userId + "_viewer_summary.csv" + val tmp = java.io.File.createTempFile(userId + "_viewer_summary", ".csv") + try { + val w = new java.io.PrintWriter(tmp, "UTF-8") + try w.write(csv) finally w.close() + CloudStorageUtil.upload(storageType, container, objectKey, tmp.getAbsolutePath) + } finally tmp.delete() + } + + // (csv header, result-row key) — getRecords/createResponse returns camelCase field names + // (courseid->courseId, completionpercentage->completionPercentage, ...), so read those keys. + private val csvCols = List( + ("courseid", "courseId"), ("batchid", "batchId"), ("progress", "progress"), + ("status", "status"), ("completionpercentage", "completionPercentage"), ("completedon", "completedOn")) + private def toCsv(rows: util.List[util.Map[String, AnyRef]]): String = { + val sb = new StringBuilder(csvCols.map(_._1).mkString(",")).append("\n") + rows.asScala.foreach { r => + sb.append(csvCols.map { case (_, key) => Option(r.get(key)).map(_.toString.replace(",", " ")).getOrElse("") }.mkString(",")).append("\n") + } + sb.toString + } + + /** Delete enrolment rows: all for the user, or a single collection[+batch]. */ + private def summaryDelete(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]) + .getOrElse(request.get("userId").asInstanceOf[String]) + val courseId = ViewerRequestKeys.courseId(request).orNull + val batchId = ViewerRequestKeys.batchId(request).orNull + + if (StringUtils.isBlank(courseId)) { + // delete all: fetch keys then delete each row + val rows = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, + new util.HashMap[String, AnyRef]() {{ put("userid", userId) }}, ctx) + rows.asScala.foreach(r => deleteEnrolment(userId, strOrNull(r.get("courseId")), strOrNull(r.get("batchId")), ctx)) + } else { + deleteEnrolment(userId, courseId, batchId, ctx) + } + logger.info(ctx, s"summary: delete | user=$userId course=${Option(courseId).getOrElse("ALL")}") + sender().tell(successResponse(), self) + } + + private def deleteEnrolment(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = { + val key = new util.HashMap[String, String]() + key.put("userid", userId) + if (StringUtils.isNotBlank(courseId)) key.put("courseid", courseId) + if (StringUtils.isNotBlank(batchId)) key.put("batchid", batchId) + cassandraOperation.deleteRecord(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, key, ctx) + } + + private def getRecords(keyspace: String, table: String, filters: util.HashMap[String, AnyRef], ctx: RequestContext): util.List[util.Map[String, AnyRef]] = { + val response = cassandraOperation.getRecords(keyspace, table, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + response.getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + } + + private def strOrNull(v: AnyRef): String = if (v == null) null else v.asInstanceOf[String] + + // for tests + def setCassandraOperation(ops: org.sunbird.cassandra.CassandraOperation): ViewerSummaryActor = { + cassandraOperation = ops + this + } +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/EnrolDispatcher.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/EnrolDispatcher.scala new file mode 100644 index 00000000..d29c715d --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/EnrolDispatcher.scala @@ -0,0 +1,53 @@ +package org.sunbird.viewer.engine + +import org.apache.pekko.actor.{ActorContext, ActorRef} +import org.sunbird.common.ProjectUtil +import org.sunbird.http.HttpClientUtil +import org.sunbird.keys.JsonKey +import org.sunbird.logging.LoggerUtil +import org.sunbird.request.{Request, RequestContext} + +import java.util + +/** System enrol via the standard `enrol` op; transport picked by deployment_mode (monolith tell vs HTTP POST). + * requestId="system" is stored as addedBy; re-enrol safety is the caller's. */ +trait EnrolDispatcher { + def enrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit +} + +object EnrolDispatcher { + private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) // default monolith + def apply(context: ActorContext): EnrolDispatcher = + if (isMonolith) new MonolithEnrolDispatcher(context) else new HttpEnrolDispatcher() +} + +/** In-JVM: tell the enrolment actor. noSender so its success reply goes to deadLetters, not the caller. */ +class MonolithEnrolDispatcher(context: ActorContext) extends EnrolDispatcher { + private val logger = new LoggerUtil(classOf[MonolithEnrolDispatcher]) + override def enrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = { + val req = new Request() + req.setRequestContext(ctx) + req.setRequestId("system") + req.setOperation("enrol") + req.put(JsonKey.USER_ID, userId); req.put(JsonKey.COURSE_ID, courseId); req.put(JsonKey.BATCH_ID, batchId) + val path = Option(ProjectUtil.getConfigValue("enrolment_actor_path")).filter(_.nonEmpty).getOrElse("/user/course-enrolment-actor") + context.actorSelection(path).tell(req, ActorRef.noSender) + logger.info(ctx, s"viewer.lp: enrol dispatched | user=$userId course=$courseId batch=$batchId mode=monolith") + } +} + +/** Distributed: POST the full enrol op to the enrolment service, authenticated with the system token. */ +class HttpEnrolDispatcher extends EnrolDispatcher { + private val logger = new LoggerUtil(classOf[HttpEnrolDispatcher]) + override def enrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = { + val base = Option(ProjectUtil.getConfigValue("enrolment_service_base_url")).filter(_.nonEmpty).getOrElse("http://lern-service:9000") + val body = s"""{"request":{"userId":"$userId","courseId":"$courseId","batchId":"$batchId"}}""" + val headers = new util.HashMap[String, String]() {{ + put("Content-Type", "application/json") + Option(ProjectUtil.getConfigValue("viewer_system_auth_token")).filter(_.nonEmpty) + .foreach(t => put("x-authenticated-user-token", t)) + }} + HttpClientUtil.post(base + "/v1/course/enroll", body, headers, ctx) + logger.info(ctx, s"viewer.lp: enrol dispatched | user=$userId course=$courseId batch=$batchId mode=distributed") + } +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala new file mode 100644 index 00000000..ae6cd46c --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala @@ -0,0 +1,189 @@ +package org.sunbird.viewer.engine + +import org.apache.commons.collections4.CollectionUtils +import org.sunbird.activity.util.{CertificateUtil, LpPolicyUtil} +import org.sunbird.assessment.service.CassandraService +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.keys.JsonKey +import org.sunbird.logging.LoggerUtil +import org.sunbird.request.RequestContext +import org.sunbird.viewer.util.ProgressionPolicy + +import java.util +import scala.collection.JavaConverters._ + +/** LP progression: compute optionality once, open the next course(s) level-by-level via EnrolDispatcher, + * credit skills at completion. Pure decisions in ProgressionPolicy; caller supplies status + ancestorsOf. */ +class LpProgressionEngine(cassandraOperation: CassandraOperation, + enrolKeyspace: String, enrolTable: String, + lpPolicyUtil: LpPolicyUtil, + assessmentService: CassandraService, + dispatcher: EnrolDispatcher, + certificateUtil: CertificateUtil) { + + private val logger = new LoggerUtil(classOf[LpProgressionEngine]) + private val USER_SKILLS_TABLE = "user_skills" + + // status: (courseId,batchId)->status for all the user's enrolments; ancestorsOf: course->chain (root LAST). + def advance(userId: String, rootId: String, batchId: String, trackable: List[String], + status: Map[(String, String), Int], ancestorsOf: String => List[String], ctx: RequestContext): Unit = { + val childBatchOf = (c: String) => batchId + ":" + c + val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) + + val levelByCourse = ProgressionPolicy.levelByCourse(trackable, ancestorsOf, rootId) // ancestorsOf runs once per course + // Halt only when optionality can't be established (Adaptive misconfigured: no pre-assessment) — open nothing. + if (!ensureOptionalityComputed(userId, rootId, batchId, trackable, levelByCourse, status, ctx)) return + val optional = readOptionalNodes(userId, rootId, batchId, ctx).toSet + val levels = ProgressionPolicy.orderedLevels(trackable, levelByCourse) + + // Level complete = all its required (non-optional) courses complete (empty required set = complete, §5). + def levelComplete(level: String): Boolean = + ProgressionPolicy.coursesOfLevel(level, trackable, levelByCourse).filterNot(optional.contains).forall(courseComplete) + + // Open first incomplete level: enrol all its optionals up front + the next single required course (§5). + levels.find(l => !levelComplete(l)).foreach { level => + val courses = ProgressionPolicy.coursesOfLevel(level, trackable, levelByCourse) + val nextRequired = courses.filterNot(optional.contains).find(c => !courseComplete(c)) + val toOpen = courses.filter(optional.contains) ++ nextRequired.toList + logger.info(ctx, s"viewer.lp: level opened | user=$userId root=$rootId level=$level open=[${toOpen.mkString(",")}]") + toOpen.foreach { c => + val childBatch = childBatchOf(c) + if (!status.contains((c, childBatch))) dispatcher.enrol(userId, c, childBatch, ctx) + else logger.info(ctx, s"viewer.lp: enrol skip(already) | user=$userId course=$c batch=$childBatch") + } + } + + // Root progress/status/cert from trackable-course completion: the root's own leaf consumption is empty + // (leaves are consumed under child batches), so derive its progress from how many required courses are done. + val requiredCourses = trackable.filterNot(optional.contains) + val allComplete = levels.nonEmpty && levels.forall(levelComplete) + writeRootProgress(userId, rootId, batchId, requiredCourses.count(courseComplete), requiredCourses.size, + allComplete, status.get((rootId, batchId)).getOrElse(0), ctx) + + if (allComplete) { + logger.info(ctx, s"viewer.lp: complete | user=$userId root=$rootId") + creditSkills(userId, rootId, batchId, trackable, ctx) + } + } + + // Update the LP-root user_enrolments row (progress/%/status; completedon on the 2-transition) and fire the + // LP certificate once when status first reaches 2. Partial update — does not touch contentstatus. + private def writeRootProgress(userId: String, rootId: String, batchId: String, done: Int, total: Int, + allComplete: Boolean, currentStatus: Int, ctx: RequestContext): Unit = { + val pct = if (total <= 0) 100 else math.min(100, done * 100 / total) + val rootStatus = if (allComplete) 2 else if (done > 0) 1 else 0 + val select = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", rootId); put("batchid", batchId) }} + val update = new util.HashMap[String, AnyRef]() {{ + put("progress", Integer.valueOf(done)); put("completionpercentage", Integer.valueOf(pct)); put("status", Integer.valueOf(rootStatus)) + if (rootStatus == 2 && currentStatus != 2) put("completedon", new java.util.Date()) + }} + cassandraOperation.updateRecordV2(enrolKeyspace, enrolTable, select, update, true, ctx) + logger.info(ctx, s"viewer.lp: root progress | user=$userId root=$rootId done=$done/$total pct=$pct status=$rootStatus") + if (rootStatus == 2 && currentStatus != 2) certificateUtil.publishCertificateIssueEvent(userId, rootId, batchId, ctx) + } + + // Returns whether advance() may open courses; false ONLY for a misconfigured Adaptive LP (no pre-assessment). + private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], + levelByCourse: Map[String, String], + status: Map[(String, String), Int], ctx: RequestContext): Boolean = { + if (optionalityComputed(userId, rootId, batchId, ctx)) return true + val meta = lpPolicyUtil.lpMeta(rootId, ctx) + val policy = lpPolicyUtil.policyOf(meta) + if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { + logger.info(ctx, s"viewer.lp: optionality computed(strict) optional=[] | user=$userId root=$rootId") + writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return true + } + // pre-assessment = an assessment course in the FIRST level (seeds achieved skills before waivers). + val levels = ProgressionPolicy.orderedLevels(trackable, levelByCourse) + val preAssessment = ProgressionPolicy.coursesOfLevel(levels.headOption.getOrElse(""), trackable, levelByCourse) + .find(c => lpPolicyUtil.isAssessmentCourse(c, meta)) + if (policy.equalsIgnoreCase("Adaptive") && preAssessment.isEmpty) { + logger.warn(ctx, s"viewer.lp: Adaptive LP has no pre-assessment; halting (misconfigured, opening nothing) | user=$userId root=$rootId", null) + return false + } + val childBatchOf = (c: String) => batchId + ":" + c + val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) + // Pre-assessment not done yet: don't compute waivers, but let advance open it (it's the first-level course). + if (preAssessment.exists(pa => !courseComplete(pa))) return true + val achieved = preAssessment.map(pa => skillsFromAssessment(userId, rootId, pa, batchId, ctx)).getOrElse(Set.empty) + // PriorLearning: a content course already completed under ANY batch is optional directly. + val completedCourses = status.collect { case ((c, _), 2) => c }.toSet // O(status) once, not O(trackable×status) + val priorCompleted = + if (policy.equalsIgnoreCase("PriorLearning")) trackable.filter(completedCourses.contains).toSet + else Set.empty[String] + val cMeta = lpPolicyUtil.courseMeta(trackable, meta) + val assessmentCourses = cMeta.collect { case (c, (_, true)) => c }.toSet + val skillsByCourse = cMeta.map { case (c, (s, _)) => c -> s } + logger.info(ctx, s"viewer.lp: optionality computing($policy) preAssess=${preAssessment.getOrElse("-")} achieved=${achieved.size} priorDone=${priorCompleted.size} | user=$userId root=$rootId") + writeOptionalNodes(userId, rootId, batchId, + ProgressionPolicy.computeOptionalNodes(policy, trackable, skillsByCourse, assessmentCourses, achieved, priorCompleted), ctx) + true + } + + private def isAssessmentCourse(rootId: String, courseId: String, ctx: RequestContext): Boolean = + lpPolicyUtil.isAssessmentCourse(courseId, lpPolicyUtil.lpMeta(rootId, ctx)) + + // terms of the questions the learner got fully correct in the pre-assessment's question set(s). + private def skillsFromAssessment(userId: String, rootId: String, courseId: String, batchId: String, ctx: RequestContext): Set[String] = { + val meta = lpPolicyUtil.lpMeta(rootId, ctx) + val childBatch = batchId + ":" + courseId + val correct = lpPolicyUtil.questionSetsOf(courseId, meta).flatMap { qs => + val attempts = assessmentService.getUserAssessments(userId, courseId, childBatch, qs, ctx) + if (attempts.isEmpty) Nil + else attempts.maxBy(_.totalScore).questions.collect { case q if q.maxScore > 0 && q.score == q.maxScore => q.questionId } + }.distinct + lpPolicyUtil.skillsOfQuestions(correct, meta) + } + + // Durable skill credit — ONCE at LP completion. Read-union-upsert (portable; no set-append needed). + private def creditSkills(userId: String, rootId: String, batchId: String, trackable: List[String], ctx: RequestContext): Unit = { + val earned = trackable.filter(c => isAssessmentCourse(rootId, c, ctx)).flatMap(c => skillsFromAssessment(userId, rootId, c, batchId, ctx)).toSet + if (earned.isEmpty) return + val existing = readUserSkills(userId, ctx) + val merged = existing ++ earned + if (merged.size == existing.size) return // nothing new — credit already granted + val row = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("skills", merged.asJava) }} + cassandraOperation.insertRecord(enrolKeyspace, USER_SKILLS_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + logger.info(ctx, s"LpProgressionEngine: credited ${earned.size} skills to user=$userId for LP=$rootId") + } + + private def writeOptionalNodes(userId: String, rootId: String, batchId: String, optional: Set[String], ctx: RequestContext): Unit = { + val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", rootId); put("batchid", batchId) }} + val updateMap = new util.HashMap[String, AnyRef]() {{ put("optional_nodes", optional.asJava) }} + cassandraOperation.updateRecordV2(enrolKeyspace, enrolTable, selectMap, updateMap, true, ctx) + LpProgressionEngine.markOptionalityComputed(userId, rootId, batchId) // remember empty results too (no DB column) + } + + /** Computed? Non-empty optional_nodes is self-evident; an empty result is remembered in an in-process memo. */ + private def optionalityComputed(userId: String, rootId: String, batchId: String, ctx: RequestContext): Boolean = + readOptionalNodes(userId, rootId, batchId, ctx).nonEmpty || + LpProgressionEngine.isOptionalityComputed(userId, rootId, batchId) + + private def readOptionalNodes(userId: String, rootId: String, batchId: String, ctx: RequestContext): List[String] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", rootId); put("batchid", batchId) }} + val rows = cassandraOperation.getRecords(enrolKeyspace, enrolTable, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) + Option(rows.get(0).get("optional_nodes")).map(_.asInstanceOf[util.Collection[String]].asScala.toList).getOrElse(List()) + else List() + } + + private def readUserSkills(userId: String, ctx: RequestContext): Set[String] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} + val rows = cassandraOperation.getRecords(enrolKeyspace, USER_SKILLS_TABLE, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) + Option(rows.get(0).get("skills")).map(_.asInstanceOf[util.Collection[String]].asScala.toSet).getOrElse(Set.empty) + else Set.empty + } +} + +object LpProgressionEngine { + // JVM-wide memo so empty-optionality isn't recomputed each pass. ponytail: unbounded; add a cap if it grows. + private val optionalityDone: java.util.Set[String] = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() + private def optKey(userId: String, rootId: String, batchId: String): String = s"$userId:$rootId:$batchId" + def markOptionalityComputed(userId: String, rootId: String, batchId: String): Unit = + optionalityDone.add(optKey(userId, rootId, batchId)) + def isOptionalityComputed(userId: String, rootId: String, batchId: String): Boolean = + optionalityDone.contains(optKey(userId, rootId, batchId)) +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala new file mode 100644 index 00000000..4150a269 --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala @@ -0,0 +1,54 @@ +package org.sunbird.viewer.util + +/** + * Pure, host-agnostic decisions for Learning-Path progression. No I/O — every input is passed in, + * so these are trivially unit-testable and run identically in-request or in the async aggregator. + */ +object ProgressionPolicy { + + /** + * The course's level = the ancestor that is a direct child of the root (top-most under root), + * i.e. lastOption of the ancestor chain excluding the root — NOT the nearest ancestor. `ancestorsOf` + * returns the chain nearest-first. + */ + def levelOf(course: String, ancestorsOf: String => List[String], root: String): Option[String] = + ancestorsOf(course).filterNot(_ == root).lastOption + + /** Trackable courses whose level is `level`, preserving `trackable` (trackablenodes) order. */ + def coursesOfLevel(level: String, trackable: List[String], + ancestorsOf: String => List[String], root: String): List[String] = + trackable.filter(c => levelOf(c, ancestorsOf, root).contains(level)) + + /** Distinct level nodes in first-appearance order along `trackable`. */ + def orderedLevels(trackable: List[String], + ancestorsOf: String => List[String], root: String): List[String] = + trackable.flatMap(c => levelOf(c, ancestorsOf, root)).distinct + + /** Precompute each course's level once so downstream lookups don't re-invoke ancestorsOf per call. */ + def levelByCourse(trackable: List[String], ancestorsOf: String => List[String], root: String): Map[String, String] = + trackable.flatMap(c => levelOf(c, ancestorsOf, root).map(c -> _)).toMap + + def orderedLevels(trackable: List[String], levelByCourse: Map[String, String]): List[String] = + trackable.flatMap(levelByCourse.get).distinct + + def coursesOfLevel(level: String, trackable: List[String], levelByCourse: Map[String, String]): List[String] = + trackable.filter(c => levelByCourse.get(c).contains(level)) + + /** + * A course is optional iff it is not an assessment and all of its (non-empty) skills are achieved. + * `Strict` waives nothing; other policies differ only in how the caller builds `skillsAchieved`. + */ + def computeOptionalNodes(policy: String, courses: List[String], + skillsByCourse: Map[String, Set[String]], + assessmentCourses: Set[String], + skillsAchieved: Set[String], + priorCompleted: Set[String] = Set.empty): Set[String] = { + if ("Strict".equalsIgnoreCase(policy)) Set.empty + else courses.filter { c => + !assessmentCourses.contains(c) && { + val skills = skillsByCourse.getOrElse(c, Set.empty) + priorCompleted.contains(c) || (skills.nonEmpty && skills.subsetOf(skillsAchieved)) + } + }.toSet + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala new file mode 100644 index 00000000..a43f4e6c --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala @@ -0,0 +1,192 @@ +package org.sunbird.viewer.actor + +import java.util +import java.util.concurrent.TimeUnit + +import org.apache.pekko.actor.{Actor, ActorSystem, Props} +import org.apache.pekko.testkit.TestKit +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.response.Response + +import scala.concurrent.ExecutionContext +import scala.concurrent.duration.FiniteDuration + +/** + * Unit tests for ViewConsumptionActor — view lifecycle (start/update/end/read) + assessment + * (submit no-events branch + read). CassandraOperation is mocked via the setCassandraOperation seam; + * the aggregator is a stub actor that replies immediately so the sync ask in viewEnd/viewAssess + * returns fast. Scoring math itself is covered by AssessmentServiceSpec, not re-tested here. + * + * Every write op also calls touchEnrolmentAccess, which reads the enrolment (getRecordByIdentifier) + * and only stamps it (updateRecordV2) when it EXISTS — so each test stubs that read. The two + * "touchEnrolmentAccess" cases pin that guard (C1: no phantom enrolment row on an unenrolled view). + */ +class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactory { + + implicit val ec: ExecutionContext = ExecutionContext.global + val system: ActorSystem = ActorSystem.create("viewer-consumption-test") + + // stub aggregator: replies to the Patterns.ask so triggerAggregation completes without the 30s timeout + private def replyingAggregator = system.actorOf(Props(new Actor { + def receive: Receive = { case _ => sender() ! new Response() } + })) + + private def emptyRows: Response = { + val r = new Response(); r.put("response", new util.ArrayList[util.Map[String, AnyRef]]()); r + } + + private def rowsWith(rows: util.List[util.Map[String, AnyRef]]): Response = { + val r = new Response(); r.put("response", rows); r + } + + private def uccRow(status: Int): util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() {{ + put("userid", "u1"); put("collectionid", "c1"); put("contextid", "b1"); put("contentid", "ct1") + put("status", Integer.valueOf(status)) + }} + + private def enrolmentRow: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() {{ + put("userid", "u1"); put("courseId", "c1"); put("batchId", "b1"); put("status", Integer.valueOf(1)) + }} + + // touchEnrolmentAccess's enrolment read; `result` decides whether the row is stamped. + private def stubEnrolmentRead(ops: CassandraOperation, result: Response) = + (ops.getRecordByIdentifier(_: String, _: String, _: Object, _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(result) + + private def expectEnrolmentStamp(ops: CassandraOperation) = + (ops.updateRecordV2(_: String, _: String, _: util.Map[String, AnyRef], _: util.Map[String, AnyRef], _: Boolean, _: RequestContext)) + .expects(*, *, *, *, *, *).returns(new Response()).once() + + private def callActor(request: Request, props: Props): Response = { + val probe = new TestKit(system) + val actorRef = system.actorOf(props) + actorRef.tell(request, probe.testActor) + probe.expectMsgType[Response](FiniteDuration.apply(15, TimeUnit.SECONDS)) + } + + private def viewRequest(op: String): Request = { + val req = new Request + req.setOperation(op) + req.put("userId", "u1"); req.put("courseId", "c1"); req.put("batchId", "b1"); req.put("contentId", "ct1") + req + } + + "viewStart" should "insert a new ucc row when absent" in { + val ops = mock[CassandraOperation] + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows) + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) + val result = callActor(viewRequest("viewStart"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + "viewStart" should "be a no-op when the row already exists" in { + val ops = mock[CassandraOperation] + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(uccRow(1)) + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(rowsWith(rows)) + // no upsertRecord expectation -> a call would fail the strict mock + stubEnrolmentRead(ops, emptyRows) + val result = callActor(viewRequest("viewStart"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + "viewUpdate" should "upsert when the row exists and is not completed" in { + val ops = mock[CassandraOperation] + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(uccRow(1)) + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(rowsWith(rows)) + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) + val result = callActor(viewRequest("viewUpdate"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + "viewEnd" should "write status=2 and trigger the aggregation" in { + val ops = mock[CassandraOperation] + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) + val result = callActor(viewRequest("viewEnd"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + // C1: touchEnrolmentAccess must NOT fabricate an enrolment for an unenrolled/no-context view. + "touchEnrolmentAccess" should "not stamp the enrolment when none exists" in { + val ops = mock[CassandraOperation] + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) // no enrolment + // no updateRecordV2 expectation -> a stamp write would fail the strict mock (phantom-row guard) + val result = callActor(viewRequest("viewEnd"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + // C1: when the enrolment DOES exist, its last-access is stamped exactly once. + "touchEnrolmentAccess" should "stamp the enrolment when it exists" in { + val ops = mock[CassandraOperation] + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(enrolmentRow) + stubEnrolmentRead(ops, rowsWith(rows)) // enrolment present + expectEnrolmentStamp(ops) + val result = callActor(viewRequest("viewEnd"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + "viewRead" should "return the ucc rows" in { + val ops = mock[CassandraOperation] + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(uccRow(2)) + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(rowsWith(rows)) + val result = callActor(viewRequest("viewRead"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + val out = result.getResult.get("response").asInstanceOf[util.List[util.Map[String, AnyRef]]] + out.size() shouldBe 1 + } + + "viewAssess" should "mark the content complete and trigger aggregation when no events are sent" in { + val ops = mock[CassandraOperation] + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) + val result = callActor(viewRequest("viewAssess"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result.getResult.get("ct1") shouldBe "SUCCESS" + } + + "assessmentRead" should "return best score/max score per content" in { + val ops = mock[CassandraOperation] + val attempts = new util.ArrayList[util.Map[String, AnyRef]]() + attempts.add(new util.HashMap[String, AnyRef]() {{ + put("attempt_id", "a1"); put("content_id", "ct1") + put("total_score", java.lang.Double.valueOf(6.0)); put("total_max_score", java.lang.Double.valueOf(10.0)) + }}) + attempts.add(new util.HashMap[String, AnyRef]() {{ + put("attempt_id", "a2"); put("content_id", "ct1") + put("total_score", java.lang.Double.valueOf(8.0)); put("total_max_score", java.lang.Double.valueOf(10.0)) + }}) + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(rowsWith(attempts)) + val result = callActor(viewRequest("assessmentRead"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + val contents = result.getResult.get("contents").asInstanceOf[util.List[util.Map[String, AnyRef]]] + contents.size() shouldBe 1 + contents.get(0).get("identifier") shouldBe "ct1" + // best attempt (8.0) wins + contents.get(0).get("score").asInstanceOf[Double] shouldBe 8.0 + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala new file mode 100644 index 00000000..80a5dc39 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala @@ -0,0 +1,135 @@ +package org.sunbird.viewer.actor + +import java.util +import java.util.concurrent.TimeUnit + +import org.apache.pekko.actor.{ActorSystem, Props} +import org.apache.pekko.testkit.TestKit +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.activity.util.{CertificateUtil, HierarchyRelationsUtil} +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.response.Response + +import scala.concurrent.duration.FiniteDuration + +/** + * Unit tests for ViewerAggregatorActor guard branches (deterministic, no hierarchy fixture needed): + * - missing userId/collectionId -> skip, still replies success. + * - no consumption rows -> early return, NO aggregate/enrolment writes. + * The full recursive-rollup happy path depends on a published hierarchy_relations fixture and is + * better exercised as an integration test; these guards pin the cheap-exit correctness. + */ +class ViewerAggregatorActorTest extends AnyFlatSpec with Matchers with MockFactory { + + val system: ActorSystem = ActorSystem.create("viewer-aggregator-test") + + private def emptyRows: Response = { + val r = new Response(); r.put("response", new util.ArrayList[util.Map[String, AnyRef]]()); r + } + + private def callActor(request: Request, props: Props): Response = { + val probe = new TestKit(system) + val actorRef = system.actorOf(props) + actorRef.tell(request, probe.testActor) + probe.expectMsgType[Response](FiniteDuration.apply(15, TimeUnit.SECONDS)) + } + + private def aggRequest(userId: String, courseId: String): Request = { + val req = new Request + req.setOperation("aggregate") + if (userId != null) req.put("userId", userId) + if (courseId != null) req.put("courseId", courseId) + req.put("batchId", "b1") + req + } + + "aggregate" should "skip and reply success when userId/courseId are missing" in { + val ops = mock[CassandraOperation] + val hru = mock[HierarchyRelationsUtil] + val cu = mock[CertificateUtil] + // no cassandra / hierarchy / cert interaction expected on the missing-id guard + val result = callActor(aggRequest(null, null), Props(new ViewerAggregatorActor().configure(ops, hru, cu))) + result should not be null + } + + "aggregate" should "early-return with no writes when there is no consumption" in { + val ops = mock[CassandraOperation] + val hru = mock[HierarchyRelationsUtil] + val cu = mock[CertificateUtil] + // not an LP (empty trackablenodes) -> plain rollup path; no consumption -> early return, no writes + (hru.getTrackableNodes(_: String, _: RequestContext)).expects(*, *).returns(List()) + // readConsumption -> empty; must NOT reach batchUpdateWithPutAll / updateRecordV2 / cert + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows) + val result = callActor(aggRequest("u1", "c1"), Props(new ViewerAggregatorActor().configure(ops, hru, cu))) + result should not be null + } + + // C2: contentstatus is a full-column replace in updateRecordV2, so the rollup must MERGE freshly computed + // leaf statuses into the enrolment row's existing map — a root-keyed read that sees only some leaves must + // not wipe the others. Tests the extracted pure merge directly (the full rollup is an integration concern). + "mergeContentStatus" should "preserve existing leaves and add/overwrite the fresh ones" in { + val existing = new util.HashMap[String, AnyRef]() {{ + put("leaf-a", Integer.valueOf(2)) // completed earlier, not in this pass + put("leaf-b", Integer.valueOf(1)) // in-progress, gets overwritten below + }} + val fresh = Map[String, AnyRef]("leaf-b" -> Integer.valueOf(2), "leaf-c" -> Integer.valueOf(2)) + val merged = ViewerAggregatorActor.mergeContentStatus(existing, fresh) + merged.get("leaf-a") shouldBe Integer.valueOf(2) // preserved (not clobbered) + merged.get("leaf-b") shouldBe Integer.valueOf(2) // fresh wins on conflict + merged.get("leaf-c") shouldBe Integer.valueOf(2) // added + merged.size() shouldBe 3 + } + + "mergeContentStatus" should "tolerate a null existing map" in { + val merged = ViewerAggregatorActor.mergeContentStatus(null, Map[String, AnyRef]("leaf-a" -> Integer.valueOf(1))) + merged.get("leaf-a") shouldBe Integer.valueOf(1) + merged.size() shouldBe 1 + } + + // resolveParentLp: the completion trigger's parent-lookup. Child batch = "LPbatch:childId"; the LP is the + // course_batch row keyed by the stripped LPbatch whose courseid differs from the completed course. + private def batchRow(courseId: String): util.Map[String, AnyRef] = + new util.HashMap[String, AnyRef]() {{ put("courseId", courseId) }} + private def batchRows(courseIds: String*): util.List[util.Map[String, AnyRef]] = { + val l = new util.ArrayList[util.Map[String, AnyRef]](); courseIds.foreach(c => l.add(batchRow(c))); l + } + + "resolveParentLp" should "return None for a standalone (colon-free) batch without fetching" in { + var fetched = false + val out = ViewerAggregatorActor.resolveParentLp("course-1", "plainBatch", _ => { fetched = true; batchRows() }) + out shouldBe None + fetched shouldBe false // standalone course -> no course_batch read, no trigger + } + + "resolveParentLp" should "resolve the LP course + stripped LP batch from the child batch" in { + val out = ViewerAggregatorActor.resolveParentLp( + "course-1", "lpBatch-9:course-1", lpBatch => { lpBatch shouldBe "lpBatch-9"; batchRows("course-1", "lp-root") }) + out shouldBe Some(("lp-root", "lpBatch-9")) // the row whose courseid != the completed course + } + + "resolveParentLp" should "return None when the batch maps only to the course itself" in { + // course consumed under its own colon-batch, but course_batch has no differing (LP) row -> not an LP child + val out = ViewerAggregatorActor.resolveParentLp("course-1", "lpBatch-9:course-1", _ => batchRows("course-1")) + out shouldBe None + } + + "resolveParentLp" should "pick the first differing course id when several rows share the LP batch" in { + // In the live schema only the LP-root row is keyed by the bare lpBatch (children are lpBatch:childId), so + // this is a defensive/bad-data case: with multiple differing rows the resolver is first-match-wins. + val out = ViewerAggregatorActor.resolveParentLp("course-1", "lpBatch-9:course-1", + _ => batchRows("course-1", "lp-root", "lp-other")) + out shouldBe Some(("lp-root", "lpBatch-9")) + } + + // parseCourseCertEnabled: course certs default ON; only literal "false" disables (LP cert unaffected). + "parseCourseCertEnabled" should "default to true when unset/blank and honor an explicit false" in { + ViewerAggregatorActor.parseCourseCertEnabled(null) shouldBe true + ViewerAggregatorActor.parseCourseCertEnabled("") shouldBe true + ViewerAggregatorActor.parseCourseCertEnabled("true") shouldBe true + ViewerAggregatorActor.parseCourseCertEnabled("FALSE") shouldBe false + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala new file mode 100644 index 00000000..6f487de5 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala @@ -0,0 +1,121 @@ +package org.sunbird.viewer.engine + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.activity.util.{CertificateUtil, LpMeta, LpPolicyUtil, NodeMeta} +import org.sunbird.assessment.service.CassandraService +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.RequestContext +import org.sunbird.response.Response + +import java.util +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer + +class LpProgressionEngineSpec extends AnyFlatSpec with Matchers with MockFactory { + + private val ctx = new RequestContext() + + // 2-level LP: crsA under L1, crsB under L2 (ancestors nearest-first, root LAST). + private val ancestorsOf: String => List[String] = { + case "crsA" => List("L1", "lp") + case "crsB" => List("L2", "lp") + case _ => Nil + } + private val trackable = List("crsA", "crsB") + + private def emptyRows: Response = { val r = new Response(); r.put("response", new util.ArrayList[util.Map[String, AnyRef]]()); r } + private def optionalRows(opt: String*): Response = { + val row = new util.HashMap[String, AnyRef](); row.put("optional_nodes", opt.toList.asJava) + val list = new util.ArrayList[util.Map[String, AnyRef]](); list.add(row) + val r = new Response(); r.put("response", list); r + } + + /** Fake transport: records (user, course, batch) instead of messaging/HTTP. */ + private class FakeDispatcher extends EnrolDispatcher { + val enrolled: ListBuffer[(String, String, String)] = ListBuffer.empty + override def enrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = + enrolled += ((userId, courseId, batchId)) + } + + private def engineWith(ops: CassandraOperation, lp: LpPolicyUtil, disp: EnrolDispatcher, cert: CertificateUtil): LpProgressionEngine = + new LpProgressionEngine(ops, "ks", "user_enrolments", lp, new CassandraService(Some(ops)), disp, cert) + + // updateRecordV2 stub (writeOptionalNodes + writeRootProgress) — accept any, return a Response. + private def stubUpdate(ops: CassandraOperation): Unit = + (ops.updateRecordV2(_: String, _: String, _: util.Map[String, AnyRef], _: util.Map[String, AnyRef], _: Boolean, _: RequestContext)) + .expects(*, *, *, *, *, *).returns(new Response()).anyNumberOfTimes() + + "advance (Strict)" should "open the first level's required course and gate the next level" in { + val ops = mock[CassandraOperation] + val lp = mock[LpPolicyUtil] + val disp = new FakeDispatcher + val cert = mock[CertificateUtil] + // optional_nodes read -> empty (not yet computed); Strict writes an empty optional set. + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows).anyNumberOfTimes() + (ops.updateRecordV2(_: String, _: String, _: util.Map[String, AnyRef], _: util.Map[String, AnyRef], _: Boolean, _: RequestContext)) + .expects(*, *, *, *, *, *).returns(new Response()).anyNumberOfTimes() + (lp.lpMeta(_: String, _: RequestContext)).expects(*, *).returns(LpMeta("Strict", "", "", Map.empty[String, NodeMeta])).anyNumberOfTimes() + (lp.policyOf(_: LpMeta)).expects(*).returns("Strict").anyNumberOfTimes() + + engineWith(ops, lp, disp, cert).advance("uA", "lp", "bA", trackable, Map.empty, ancestorsOf, ctx) + + disp.enrolled.toList shouldBe List(("uA", "crsA", "bA:crsA")) // first level's course only; crsB (L2) gated + } + + "advance" should "skip a waived course and open the next level's course" in { + val ops = mock[CassandraOperation] + val lp = mock[LpPolicyUtil] + val disp = new FakeDispatcher + val cert = mock[CertificateUtil] + // optional_nodes already = [crsA] -> optionality is 'computed' (non-empty), so ensureOptionality + // returns immediately; crsA is treated as waived, L1 is complete, L2's crsB opens. + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(optionalRows("crsA")).anyNumberOfTimes() + stubUpdate(ops) + + engineWith(ops, lp, disp, cert).advance("uB", "lp", "bB", trackable, Map.empty, ancestorsOf, ctx) + + disp.enrolled.toList shouldBe List(("uB", "crsB", "bB:crsB")) // crsA waived -> next required course opens + } + + "advance" should "mark the LP root complete + fire the cert when all required courses are done" in { + val ops = mock[CassandraOperation] + val lp = mock[LpPolicyUtil] + val disp = new FakeDispatcher + val cert = mock[CertificateUtil] + // single-course LP; the course is already complete in the snapshot -> root reaches 100%/status=2. + val oneCourse = List("crsA") + val statusDone = Map(("crsA", "bD:crsA") -> 2) // root (lp,bD) absent -> currentStatus 0 -> cert fires once + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows).anyNumberOfTimes() + (lp.lpMeta(_: String, _: RequestContext)).expects(*, *).returns(LpMeta("Strict", "", "", Map.empty[String, NodeMeta])).anyNumberOfTimes() + (lp.policyOf(_: LpMeta)).expects(*).returns("Strict").anyNumberOfTimes() + (lp.isAssessmentCourse(_: String, _: LpMeta)).expects(*, *).returns(false).anyNumberOfTimes() // creditSkills -> no assessment courses + stubUpdate(ops) + (cert.publishCertificateIssueEvent(_: String, _: String, _: String, _: RequestContext)).expects("uD", "lp", "bD", *).once() + + engineWith(ops, lp, disp, cert).advance("uD", "lp", "bD", oneCourse, statusDone, ancestorsOf, ctx) + + disp.enrolled shouldBe empty // nothing left to open; LP root completed + } + + "advance (Adaptive, no pre-assessment)" should "halt and open nothing (misconfigured)" in { + val ops = mock[CassandraOperation] + val lp = mock[LpPolicyUtil] + val disp = new FakeDispatcher + val cert = mock[CertificateUtil] + // optionality not yet computed; Adaptive policy; no course is an assessment -> no pre-assessment -> halt. + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows).anyNumberOfTimes() + (lp.lpMeta(_: String, _: RequestContext)).expects(*, *).returns(LpMeta("Adaptive", "", "", Map.empty[String, NodeMeta])).anyNumberOfTimes() + (lp.policyOf(_: LpMeta)).expects(*).returns("Adaptive").anyNumberOfTimes() + (lp.isAssessmentCourse(_: String, _: LpMeta)).expects(*, *).returns(false).anyNumberOfTimes() + + engineWith(ops, lp, disp, cert).advance("uC", "lp", "bC", trackable, Map.empty, ancestorsOf, ctx) + + disp.enrolled shouldBe empty // misconfigured Adaptive LP opens no course (no updateRecordV2 either) + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilLogicSpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilLogicSpec.scala new file mode 100644 index 00000000..05313722 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilLogicSpec.scala @@ -0,0 +1,22 @@ +package org.sunbird.viewer.util + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.activity.util.{LpPolicyUtil, NodeMeta} + +class LpPolicyUtilLogicSpec extends AnyFlatSpec with Matchers { + private val nodes = Map( + "crsA" -> NodeMeta("Course", Set("Python Programming"), List("qsA")), + "qsA" -> NodeMeta("Practice Question Set", Set.empty, Nil), + "crsB" -> NodeMeta("Course", Set("JavaScript"), Nil)) + + "isAssessment" should "be true when a child is a Practice Question Set" in { + LpPolicyUtil.isAssessment("crsA", nodes) shouldBe true + LpPolicyUtil.isAssessment("crsB", nodes) shouldBe false + } + + "questionSets" should "list the Practice-Question-Set children" in { + LpPolicyUtil.questionSets("crsA", nodes) shouldBe List("qsA") + LpPolicyUtil.questionSets("crsB", nodes) shouldBe Nil + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilParseSpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilParseSpec.scala new file mode 100644 index 00000000..a894ecb4 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilParseSpec.scala @@ -0,0 +1,40 @@ +package org.sunbird.viewer.util + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.activity.util.{LpPolicyUtil, NodeMeta} + +class LpPolicyUtilParseSpec extends AnyFlatSpec with Matchers { + + "parseFrameworkCategoryCode" should "return the highest-index category code" in { + val json = """{"result":{"framework":{"identifier":"USF","categories":[ + {"code":"board","index":1},{"code":"subject","index":2},{"code":"skill","index":3}]}}}""" + LpPolicyUtil.parseFrameworkCategoryCode(json) shouldBe Some("skill") + } + + it should "return None when there are no categories" in { + LpPolicyUtil.parseFrameworkCategoryCode("""{"result":{"framework":{"categories":[]}}}""") shouldBe None + } + + "parseLpNodes" should "map each node to its primaryCategory, terms and childNodes" in { + val json = """{"result":{"count":2,"Content":[ + {"identifier":"c1","primaryCategory":"Course","skill":["Python Programming"],"childNodes":["q1"]}, + {"identifier":"c2","primaryCategory":"Practice Question Set","childNodes":[]}]}}""" + val nodes = LpPolicyUtil.parseLpNodes(json, "skill") + nodes("c1") shouldBe NodeMeta("Course", Set("Python Programming"), List("q1")) + nodes("c2").primaryCategory shouldBe "Practice Question Set" + nodes("c2").skills shouldBe empty + } + + it should "read whichever objectType array key is present (Question), not just content" in { + val json = """{"result":{"count":1,"Question":[ + {"identifier":"q1","primaryCategory":"Practice Question Set","skill":["JavaScript"]}]}}""" + LpPolicyUtil.parseLpNodes(json, "skill")("q1").skills shouldBe Set("JavaScript") + } + + "parseField" should "read a scalar field for an identifier" in { + val json = """{"result":{"content":[{"identifier":"root","policy":"PriorLearning","framework":"USF"}]}}""" + LpPolicyUtil.parseField(json, "root", "policy") shouldBe Some("PriorLearning") + LpPolicyUtil.parseField(json, "root", "framework") shouldBe Some("USF") + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala new file mode 100644 index 00000000..01169354 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala @@ -0,0 +1,91 @@ +package org.sunbird.viewer.util + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class ProgressionPolicySpec extends AnyFlatSpec with Matchers { + + // Reference tree: L1[CRS-A] L2[CRS-B,CRS-C] L3[CRS-D,CRS-E] L4[CRS-F] (ancestors nearest-first) + private val anc: Map[String, List[String]] = Map( + "CRS-A" -> List("L1", "do_lp"), + "CRS-B" -> List("L2", "do_lp"), "CRS-C" -> List("L2", "do_lp"), + "CRS-D" -> List("L3", "do_lp"), "CRS-E" -> List("L3", "do_lp"), + "CRS-F" -> List("L4", "do_lp"), + "CRS-X" -> List("grp", "L2", "do_lp")) // nested one level deeper inside L2 + private val ancestorsOf: String => List[String] = anc.getOrElse(_, Nil) + private val order = List("CRS-A", "CRS-B", "CRS-C", "CRS-D", "CRS-E", "CRS-F") + + "levelOf" should "return the level node (the ancestor that is the direct child of the root)" in { + ProgressionPolicy.levelOf("CRS-C", ancestorsOf, "do_lp") shouldBe Some("L2") + } + + it should "pick the direct child of root even when the course is nested deeper (not the nearest ancestor)" in { + ProgressionPolicy.levelOf("CRS-X", ancestorsOf, "do_lp") shouldBe Some("L2") + } + + // Hybrid: levels are derived from a LEAF's ancestors (root LAST), which also contain the unit + course. + // levelOf must still pick the level = last non-root, regardless of the extra leading nodes. + it should "pick the level from a full leaf ancestor chain [unit, course, level, root]" in { + val leafChain: String => List[String] = _ => List("U1", "CRS-B", "L2", "do_lp") + ProgressionPolicy.levelOf("CRS-B", leafChain, "do_lp") shouldBe Some("L2") + } + + it should "treat a course directly under root (no level wrapper) as its own level" in { + val flatChain: String => List[String] = _ => List("U1", "CRS-Z", "do_lp") + ProgressionPolicy.levelOf("CRS-Z", flatChain, "do_lp") shouldBe Some("CRS-Z") + } + + "coursesOfLevel" should "group the level's courses in trackablenodes order" in { + ProgressionPolicy.coursesOfLevel("L2", order, ancestorsOf, "do_lp") shouldBe List("CRS-B", "CRS-C") + } + + "orderedLevels" should "list levels in first-appearance order" in { + ProgressionPolicy.orderedLevels(order, ancestorsOf, "do_lp") shouldBe List("L1", "L2", "L3", "L4") + } + + "levelByCourse (precomputed map)" should "match the ancestorsOf-based overloads" in { + val m = ProgressionPolicy.levelByCourse(order, ancestorsOf, "do_lp") + m shouldBe Map("CRS-A" -> "L1", "CRS-B" -> "L2", "CRS-C" -> "L2", + "CRS-D" -> "L3", "CRS-E" -> "L3", "CRS-F" -> "L4") + ProgressionPolicy.orderedLevels(order, m) shouldBe ProgressionPolicy.orderedLevels(order, ancestorsOf, "do_lp") + ProgressionPolicy.coursesOfLevel("L2", order, m) shouldBe + ProgressionPolicy.coursesOfLevel("L2", order, ancestorsOf, "do_lp") + } + + "computeOptionalNodes" should "waive a fully-known non-assessment course but never an assessment" in { + val opt = ProgressionPolicy.computeOptionalNodes( + policy = "Adaptive", + courses = List("CRS-B", "CRS-C"), + skillsByCourse = Map("CRS-B" -> Set("s1"), "CRS-C" -> Set("s2")), + assessmentCourses = Set("CRS-C"), + skillsAchieved = Set("s1", "s2")) + opt shouldBe Set("CRS-B") // CRS-C is an assessment -> never optional + } + + it should "waive nothing under Strict" in { + ProgressionPolicy.computeOptionalNodes("Strict", List("CRS-B"), + Map("CRS-B" -> Set("s1")), Set.empty, Set("s1")) shouldBe empty + } + + it should "waive a prior-completed course regardless of skills (PriorLearning)" in { + val opt = ProgressionPolicy.computeOptionalNodes( + policy = "PriorLearning", + courses = List("CRS-B", "CRS-C"), + skillsByCourse = Map("CRS-B" -> Set.empty, "CRS-C" -> Set.empty), + assessmentCourses = Set.empty, + skillsAchieved = Set.empty, + priorCompleted = Set("CRS-B")) + opt shouldBe Set("CRS-B") + } + + it should "never waive an assessment course even if prior-completed" in { + val opt = ProgressionPolicy.computeOptionalNodes( + policy = "PriorLearning", + courses = List("CRS-B"), + skillsByCourse = Map("CRS-B" -> Set.empty), + assessmentCourses = Set("CRS-B"), + skillsAchieved = Set.empty, + priorCompleted = Set("CRS-B")) + opt shouldBe empty + } +} diff --git a/modules/viewer/pom.xml b/modules/viewer/pom.xml new file mode 100644 index 00000000..4f3015f5 --- /dev/null +++ b/modules/viewer/pom.xml @@ -0,0 +1,37 @@ + + + + + org.sunbird + lern-service + 1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + viewer + pom + Viewer + Viewer service: granular view APIs + recursive collection tracking + + + + actors + service + + + + + + + com.google.code.play2-maven-plugin + play2-maven-plugin + ${play2.plugin.version} + true + + + + + diff --git a/modules/viewer/service/app/controllers/BaseController.java b/modules/viewer/service/app/controllers/BaseController.java new file mode 100644 index 00000000..28ee00ff --- /dev/null +++ b/modules/viewer/service/app/controllers/BaseController.java @@ -0,0 +1,845 @@ +package controllers; + +import org.apache.pekko.actor.ActorRef; +import org.apache.pekko.actor.ActorSelection; +import org.apache.pekko.pattern.PatternsCS; +import org.apache.pekko.util.Timeout; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import modules.ApplicationStart; +import modules.OnRequestHandler; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.response.Response; +import org.sunbird.response.ResponseParams; +import org.sunbird.operations.lms.ActorOperations; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.common.ProjectUtil; +import org.sunbird.request.HeaderParam; +import org.sunbird.request.RequestContext; +import org.sunbird.response.ResponseCode; +import org.sunbird.keys.SunbirdKey; +import org.sunbird.telemetry.util.TelemetryEvents; +import org.sunbird.telemetry.util.TelemetryWriter; +import play.libs.Json; +import play.mvc.Controller; +import play.mvc.Http; +import play.mvc.Http.Request; +import play.mvc.Result; +import play.mvc.Results; +import util.Attrs; +import util.AuthenticationHelper; + +import java.io.File; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +/** + * This controller we can use for writing some common method. + * + * @author Manzarul + */ +public class BaseController extends Controller { + + private static ObjectMapper objectMapper = new ObjectMapper(); + private static final String version = "v1"; + public static final int PEKKO_WAIT_TIME = 30; + protected Timeout timeout = new Timeout(PEKKO_WAIT_TIME, TimeUnit.SECONDS); + private static final String debugEnabled = "false"; + public static final LoggerUtil logger = new LoggerUtil(BaseController.class); + + private org.sunbird.request.Request initRequest( + org.sunbird.request.Request request, String operation, Http.Request httpRequest) { + request.setOperation(operation); + request.setRequestId(httpRequest.attrs().getOptional(Attrs.REQUEST_ID).orElse(null)); + request.setEnv(getEnvironment()); + request.setRequestContext(getRequestContext(httpRequest, request)); + request.getContext().put(JsonKey.REQUESTED_BY, httpRequest.attrs().getOptional(Attrs.USER_ID).orElse(null)); + request.getRequest().put(JsonKey.REQUESTED_BY, httpRequest.attrs().getOptional(Attrs.USER_ID).orElse(null)); + if (StringUtils.isNotBlank(httpRequest.attrs().getOptional(Attrs.REQUESTED_FOR).orElse(null))) + request.getContext().put(SunbirdKey.REQUESTED_FOR, httpRequest.attrs().get(Attrs.REQUESTED_FOR)); + request.getContext().put(JsonKey.X_AUTH_TOKEN, httpRequest.attrs().getOptional(Attrs.X_AUTH_TOKEN).orElse("")); + request = transformUserId(request); + return request; + } + + private RequestContext getRequestContext(Http.Request httpRequest, org.sunbird.request.Request request) { + RequestContext requestContext = new RequestContext( + JsonKey.SERVICE_NAME, + JsonKey.PRODUCER_NAME, + request.getContext().getOrDefault(JsonKey.ENV, "").toString(), + httpRequest.header(JsonKey.X_DEVICE_ID).orElse(null), + httpRequest.header(JsonKey.X_SESSION_ID).orElse(null), + JsonKey.PID,JsonKey.P_VERSION, null); + requestContext.setActorId(httpRequest.attrs().getOptional(Attrs.ACTOR_ID).orElse(null)); + requestContext.setActorType(httpRequest.attrs().getOptional(Attrs.ACTOR_TYPE).orElse(null)); + requestContext.setRequestId(httpRequest.attrs().getOptional(Attrs.REQUEST_ID).orElse(null)); + return requestContext; + } + + /** + * Helper method for creating and initialising a request for given operation and request body. + * + * @param operation A defined actor operation + * @param requestBodyJson Optional information received in request body (JSON) + * @return Created and initialised Request (@see {@link org.sunbird.request.Request}) + * instance. + */ + protected org.sunbird.request.Request createAndInitRequest( + String operation, JsonNode requestBodyJson, Http.Request httpRequest) { + try { + org.sunbird.request.Request request = + (org.sunbird.request.Request) + mapper.RequestMapper.mapRequest( + requestBodyJson, org.sunbird.request.Request.class); + return initRequest(request, operation, httpRequest); + } catch (Exception e) { + ProjectCommonException.throwServerErrorException(ResponseCode.SERVER_ERROR); + } + return null; + } + + /** + * Helper method for creating and initialising a request for given operation. + * + * @param operation A defined actor operation + * @return Created and initialised Request (@see {@link org.sunbird.request.Request}) + * instance. + */ + protected org.sunbird.request.Request createAndInitRequest( + String operation, Http.Request httpRequest) { + org.sunbird.request.Request request = new org.sunbird.request.Request(); + return initRequest(request, operation, httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + Http.Request httpRequest) { + return handleRequest( + actorRef, + operation, + requestBodyJson, + requestValidatorFn, + null, + null, + null, + true, + httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + java.util.function.Function requestValidatorFn, + Http.Request httpRequest) { + return handleRequest( + actorRef, operation, null, requestValidatorFn, null, null, null, false, httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + String pathId, + String pathVariable, + Http.Request httpRequest) { + return handleRequest( + actorRef, operation, null, null, pathId, pathVariable, null, false, httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + String pathId, + String pathVariable, + boolean isJsonBodyRequired, + Http.Request httpRequest) { + return handleRequest( + actorRef, + operation, + null, + null, + pathId, + pathVariable, + null, + isJsonBodyRequired, + httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + Map headers, + Http.Request httpRequest) { + return handleRequest( + actorRef, + operation, + requestBodyJson, + requestValidatorFn, + null, + null, + headers, + true, + httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + String pathId, + String pathVariable, + Http.Request httpRequest) { + return handleRequest( + actorRef, + operation, + requestBodyJson, + requestValidatorFn, + pathId, + pathVariable, + null, + true, + httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + String pathId, + String pathVariable, + Map headers, + boolean isJsonBodyRequired, + Http.Request httpRequest) { + try { + org.sunbird.request.Request request = null; + if (!isJsonBodyRequired) { + request = createAndInitRequest(operation, httpRequest); + } else { + request = createAndInitRequest(operation, requestBodyJson, httpRequest); + } + if (pathId != null) { + request.getRequest().put(pathVariable, pathId); + request.getContext().put(pathVariable, pathId); + } + if (requestValidatorFn != null) requestValidatorFn.apply(request); + if (headers != null) request.getContext().put(JsonKey.HEADER, headers); + + return actorResponseHandler(actorRef, request, timeout, null, httpRequest); + } catch (Exception e) { + logger.error( + "BaseController:handleRequest: Exception occurred with error message = " + e.getMessage(), + e); + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + protected CompletionStage handleSearchRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + String pathId, + String pathVariable, + Map headers, + String esObjectType, + Http.Request httpRequest) { + try { + org.sunbird.request.Request request = null; + if (null != requestBodyJson) { + request = createAndInitRequest(operation, requestBodyJson, httpRequest); + } else { + ProjectCommonException.throwClientErrorException(ResponseCode.invalidRequestData, null); + } + if (request != null) { + if (pathId != null) { + request.getRequest().put(pathVariable, pathId); + request.getContext().put(pathVariable, pathId); + } + if (requestValidatorFn != null) requestValidatorFn.apply(request); + if (headers != null) request.getContext().put(JsonKey.HEADER, headers); + if (StringUtils.isNotBlank(esObjectType)) { + List esObjectTypeList = new ArrayList<>(); + esObjectTypeList.add(esObjectType); + ((Map) (request.getRequest().get(JsonKey.FILTERS))) + .put(JsonKey.OBJECT_TYPE, esObjectTypeList); + } + request.getRequest().put(JsonKey.REQUESTED_BY, httpRequest.attrs().getOptional(Attrs.USER_ID).orElse(null)); + } + return actorResponseHandler(actorRef, request, timeout, null, httpRequest); + } catch (Exception e) { + logger.error( + "BaseController:handleRequest: Exception occurred with error message = " + e.getMessage(), + e); + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + /** + * This method will create failure response + * + * @param request Request + * @param code ResponseCode + * @param headerCode ResponseCode + * @return Response + */ + public static Response createFailureResponse( + Request request, ResponseCode code, ResponseCode headerCode) { + + Response response = new Response(); + response.setVer(getApiVersion(request.path())); + response.setId(getApiResponseId(request)); + response.setTs(ProjectUtil.getFormattedDate()); + response.setResponseCode(headerCode); + response.setParams(createResponseParamObj(code, null, request.attrs().getOptional(Attrs.REQUEST_ID).orElse(null))); + return response; + } + + public static ResponseParams createResponseParamObj(ResponseCode code, String customMessage, String requestId) { + ResponseParams params = new ResponseParams(); + if (code.getResponseCode() != 200) { + params.setErr(code.getErrorCode()); + params.setErrmsg( + StringUtils.isNotBlank(customMessage) ? customMessage : code.getErrorMessage()); + } + params.setMsgid(requestId); + params.setStatus(ResponseCode.getHeaderResponseCode(code.getResponseCode()).name()); + return params; + } + + /** + * This method will create data for success response. + * + * @param request play.mvc.Http.Request + * @param response Response + * @return Result + */ + public static Result createSuccessResponse(Request request, Response response) { + if (request != null) { + response.setVer(getApiVersion(request.path())); + } else { + response.setVer(""); + } + + response.setId(getApiResponseId(request)); + response.setTs(ProjectUtil.getFormattedDate()); + ResponseCode code = ResponseCode.getResponse(ResponseCode.success.getErrorCode()); + code.setResponseCode(ResponseCode.OK.getResponseCode()); + response.setParams(createResponseParamObj(code, null, request.attrs().getOptional(Attrs.REQUEST_ID).orElse(null))); + + String value = null; + try { + if (response.getResult() != null) { + String json = new ObjectMapper().writeValueAsString(response.getResult()); + value = getResponseSize(json); + } + } catch (Exception e) { + value = "0.0"; + } + + return Results.ok(Json.toJson(response)) + .withHeader(HeaderParam.X_Response_Length.getName(), value); + } + + /** + * This method will provide api version. + * + * @param request String + * @return String + */ + public static String getApiVersion(String request) { + + return request.split("[/]")[1]; + } + + /** + * This method will handle response in case of exception + * + * @param request play.mvc.Http.Request + * @param exception ProjectCommonException + * @return Response + */ + public static Response createResponseOnException( + Http.Request request, ProjectCommonException exception) { + logger.error( + (org.sunbird.request.RequestContext) null, + exception != null ? exception.getMessage() : "Message is not coming", + exception, + genarateTelemetryInfoForError(request)); + Response response = new Response(); + response.setVer(""); + if (request != null) { + response.setVer(getApiVersion(request.path())); + } + response.setId(getApiResponseId(request)); + response.setTs(ProjectUtil.getFormattedDate()); + if (exception != null) { + response.setResponseCode(ResponseCode.getHeaderResponseCode(exception.getErrorResponseCode())); + ResponseCode code = ResponseCode.getResponse(exception.getCode()); + if (code == null) { + code = ResponseCode.SERVER_ERROR; + } + response.setParams(createResponseParamObj(code, exception.getMessage(), request.attrs().getOptional(Attrs.REQUEST_ID).orElse(null))); + if (response.getParams() != null) { + response.getParams().setStatus(response.getParams().getStatus()); + if (exception.getCode() != null) { + response.getParams().setStatus(exception.getCode()); + } + if (!StringUtils.isBlank(response.getParams().getErrmsg()) + && response.getParams().getErrmsg().contains("{0}")) { + response.getParams().setErrmsg(exception.getMessage()); + } + } + } + return response; + } + + /** + * @param path String + * @param method String + * @param exception ProjectCommonException + * @return Response + */ + public static Response createResponseOnException( + String path, String method, ProjectCommonException exception) { + Response response = new Response(); + response.setVer(getApiVersion(path)); + response.setId(getApiResponseId(path, method)); + response.setTs(ProjectUtil.getFormattedDate()); + response.setResponseCode(ResponseCode.getHeaderResponseCode(exception.getErrorResponseCode())); + ResponseCode code = ResponseCode.getResponse(exception.getCode()); + response.setParams(createResponseParamObj(code, exception.getMessage(), null)); + return response; + } + + /** + * This method will create common response for all controller method + * + * @param response Object + * @param key String + * @param request play.mvc.Http.Request + * @return Result + */ + public Result createCommonResponse(Object response, String key, Http.Request request) { + Response courseResponse = (Response) response; + if (!StringUtils.isBlank(key)) { + Object value = courseResponse.getResult().get(JsonKey.RESPONSE); + courseResponse.getResult().remove(JsonKey.RESPONSE); + courseResponse.getResult().put(key, value); + } + return BaseController.createSuccessResponse(request, courseResponse); + } + + /** + * @param file + * @return + */ + public Result createFileDownloadResponse(File file) { + return Results.ok(file) + .withHeader("Content-Type", "application/x-download") + .withHeader("Content-disposition", "attachment; filename=" + file.getName()); + } + + private void removeFields(Map params, String... properties) { + for (String property : properties) { + params.remove(property); + } + } + + private String generateStackTrace(StackTraceElement[] elements) { + StringBuilder builder = new StringBuilder(""); + for (StackTraceElement element : elements) { + + builder.append(element.toString()); + builder.append("\n"); + } + return ProjectUtil.getFirstNCharacterString(builder.toString(), 100); + } + + private Map generateTelemetryRequestForController( + String eventType, Map params, Map context) { + + Map map = new HashMap<>(); + map.put(JsonKey.TELEMETRY_EVENT_TYPE, eventType); + map.put(JsonKey.CONTEXT, context); + map.put(JsonKey.PARAMS, params); + return map; + } + + /** + * Common exception response handler method. + * + * @param e Exception + * @param request play.mvc.Http.Request + * @return Result + */ + public Result createCommonExceptionResponse(Exception e, Http.Request request) { + Request req = request; + logger.error((org.sunbird.request.RequestContext) null, e.getMessage(), e, genarateTelemetryInfoForError(request)); + ProjectCommonException exception = null; + if (e instanceof ProjectCommonException) { + exception = (ProjectCommonException) e; + } else { + exception = + new ProjectCommonException( + ResponseCode.internalError.getErrorCode(), + ResponseCode.internalError.getErrorMessage(), + ResponseCode.SERVER_ERROR.getResponseCode()); + } + generateExceptionTelemetry(request, exception); + // cleaning request info ... + return Results.status( + exception.getErrorResponseCode(), + Json.toJson(createResponseOnException(req, exception))); + } + + private long calculateApiTimeTaken(Long startTime) { + + Long timeConsumed = null; + if (null != startTime) { + timeConsumed = System.currentTimeMillis() - startTime; + } + return timeConsumed; + } + + /** + * This method will make a call to Pekko actor and return promise. + * + * @param actorRef ActorSelection + * @param request Request + * @param timeout Timeout + * @param responseKey String + * @param httpReq play.mvc.Http.Request + * @return CompletionStage + */ + public CompletionStage actorResponseHandler( + Object actorRef, + org.sunbird.request.Request request, + Timeout timeout, + String responseKey, + Http.Request httpReq) { + + String operation = request.getOperation(); + + // set header to request object , setting actor type and channel headers value + // ... + setContextData(httpReq, request); + setChannelAndActorInfo(httpReq, request); + + Function function = + new Function() { + @Override + public Result apply(Object result) { + if (ActorOperations.HEALTH_CHECK.getValue().equals(request.getOperation())) { + setGlobalHealthFlag(result); + } + + if (result instanceof Response) { + Response response = (Response) result; + return createCommonResponse(response, responseKey, httpReq); + } else if (result instanceof ProjectCommonException) { + return createCommonExceptionResponse((ProjectCommonException) result, httpReq); + } else if (result instanceof File) { + return createFileDownloadResponse((File) result); + } else { + return createCommonExceptionResponse(new Exception(), httpReq); + } + } + }; + + if (actorRef instanceof ActorRef) { + return PatternsCS.ask((ActorRef) actorRef, request, timeout).thenApply(function); + } else { + return PatternsCS.ask((ActorSelection) actorRef, request, timeout).thenApply(function); + } + } + + /** + * This method will provide environment id. + * + * @return int + */ + public int getEnvironment() { + + if (ApplicationStart.env != null) { + return ApplicationStart.env.getValue(); + } + return ProjectUtil.Environment.dev.getValue(); + } + + /** + * Method to get UserId by AuthToken + * + * @param token + * @return String + */ + public String getUserIdByAuthToken(String token) { + + return AuthenticationHelper.verifyUserAccessToken(token); + } + + /** + * Method to get API response Id + * + * @param request play.mvc.Http.Request + * @return String + */ + private static String getApiResponseId(Request request) { + + String val = ""; + if (request != null) { + String path = request.path(); + if (request.method().equalsIgnoreCase(ProjectUtil.Method.GET.name())) { + val = getResponseId(path); + if (StringUtils.isBlank(val)) { + String[] splitedpath = path.split("[/]"); + path = removeLastValue(splitedpath); + val = getResponseId(path); + } + } else { + val = getResponseId(path); + } + if (StringUtils.isBlank(val)) { + val = getResponseId(path); + if (StringUtils.isBlank(val)) { + String[] splitedpath = path.split("[/]"); + path = removeLastValue(splitedpath); + val = getResponseId(path); + } + } + } + return val; + } + + /** + * Method to get API response Id + * + * @param path String + * @param method String + * @return String + */ + private static String getApiResponseId(String path, String method) { + String val = ""; + if (ProjectUtil.Method.GET.name().equalsIgnoreCase(method)) { + val = getResponseId(path); + if (StringUtils.isBlank(val)) { + String[] splitedpath = path.split("[/]"); + String tempPath = removeLastValue(splitedpath); + val = getResponseId(tempPath); + } + } else { + val = getResponseId(path); + } + return val; + } + + /** + * Method to remove last value + * + * @param splited String [] + * @return String + */ + private static String removeLastValue(String splited[]) { + + StringBuilder builder = new StringBuilder(); + if (splited != null && splited.length > 0) { + for (int i = 1; i < splited.length - 1; i++) { + builder.append("/" + splited[i]); + } + } + return builder.toString(); + } + + private static Map genarateTelemetryInfoForError(Http.Request request) { + try{ + Map map = new HashMap<>(); + String reqContext = request.attrs().getOptional(Attrs.CONTEXT).orElse(null); + Map requestInfo = + objectMapper.readValue(reqContext, new TypeReference>() {}); + Map contextInfo = (Map) requestInfo.getOrDefault(JsonKey.CONTEXT, new HashMap()); + Map params = new HashMap<>(); + params.put(JsonKey.ERR_TYPE, JsonKey.API_ACCESS); + + map.put(JsonKey.CONTEXT, contextInfo); + map.put(JsonKey.PARAMS, params); + return map; + } catch (Exception e) { + ProjectCommonException.throwServerErrorException(ResponseCode.SERVER_ERROR); + } + return Collections.emptyMap(); + } + + public void setChannelAndActorInfo( + Http.Request httpReq, org.sunbird.request.Request reqObj) { + + reqObj.getContext().put(JsonKey.CHANNEL, httpReq.attrs().getOptional(Attrs.CHANNEL).orElse(null)); + reqObj.getContext().put(JsonKey.ACTOR_ID, httpReq.attrs().getOptional(Attrs.ACTOR_ID).orElse(null)); + reqObj.getContext().put(JsonKey.ACTOR_TYPE, httpReq.attrs().getOptional(Attrs.ACTOR_TYPE).orElse(null) ); + reqObj.getContext().put(JsonKey.APP_ID, httpReq.attrs().getOptional(Attrs.APP_ID).orElse(null)); + reqObj.getContext().put(JsonKey.DEVICE_ID, httpReq.attrs().getOptional(Attrs.DEVICE_ID).orElse(null)); + reqObj + .getContext() + .put( + JsonKey.SIGNUP_TYPE, + httpReq.attrs().getOptional(Attrs.SIGNUP_TYPE).orElse(null)); // adding signup type in request context + reqObj + .getContext() + .put( + JsonKey.REQUEST_SOURCE, + httpReq.attrs().getOptional(Attrs.REQUEST_SOURCE).orElse(null)); // ADDING Source under params in context + } + + public Map getAllRequestHeaders(Request request) { + Map map = new HashMap<>(); + Map> headers = request.getHeaders().toMap(); + Iterator>> itr = headers.entrySet().iterator(); + while (itr.hasNext()) { + Map.Entry> entry = itr.next(); + map.put(entry.getKey(), entry.getValue().get(0)); + } + return map; + } + + @SuppressWarnings("unchecked") + private void setGlobalHealthFlag(Object result) { + if (result instanceof Response) { + Response response = (Response) result; + if (Boolean.parseBoolean(ProjectUtil.getConfigValue(JsonKey.SUNBIRD_HEALTH_CHECK_ENABLE)) + && ((HashMap) response.getResult().get(JsonKey.RESPONSE)) + .containsKey(JsonKey.Healthy)) { + OnRequestHandler.isServiceHealthy = + (boolean) + ((HashMap) response.getResult().get(JsonKey.RESPONSE)) + .get(JsonKey.Healthy); + } + } else { + OnRequestHandler.isServiceHealthy = false; + } + logger.debug( + "BaseController:setGlobalHealthFlag: isServiceHealthy = " + + OnRequestHandler.isServiceHealthy); + } + + protected String getQueryString(Map queryStringMap) { + return queryStringMap + .entrySet() + .stream() + .map(p -> p.getKey() + "=" + String.join(",", p.getValue())) + .reduce((p1, p2) -> p1 + "&" + p2) + .map(s -> "?" + s) + .orElse(""); + } + + public static String getResponseSize(String response) throws UnsupportedEncodingException { + if (StringUtils.isNotBlank(response)) { + return response.getBytes("UTF-8").length + ""; + } + return "0.0"; + } + + public org.sunbird.request.Request transformUserId( + org.sunbird.request.Request request) { + if (request != null && request.getRequest() != null) { + String id = (String) request.getRequest().get(JsonKey.ID); + request.getRequest().put(JsonKey.ID, ProjectUtil.getLmsUserId(id)); + id = (String) request.getRequest().get(JsonKey.USER_ID); + request.getRequest().put(JsonKey.USER_ID, ProjectUtil.getLmsUserId(id)); + return request; + } + return request; + } + + /** + * Method to get the response id on basis of request path. + * + * @param requestPath + * @return + */ + public static String getResponseId(String requestPath) { + + String path = requestPath; + final String ver = "/" + version; + final String ver2 = "/" + JsonKey.VERSION_2; + path = path.trim(); + StringBuilder builder = new StringBuilder(""); + if (path.startsWith(ver) || path.startsWith(ver2)) { + String requestUrl = (path.split("\\?"))[0]; + if (requestUrl.contains(ver)) { + requestUrl = requestUrl.replaceFirst(ver, "api"); + } else if (requestUrl.contains(ver2)) { + requestUrl = requestUrl.replaceFirst(ver2, "api"); + } + + String[] list = requestUrl.split("/"); + for (String str : list) { + if (str.matches("[A-Za-z]+")) { + builder.append(str).append("."); + } + } + builder.deleteCharAt(builder.length() - 1); + } else { + if ("/health".equalsIgnoreCase(path)) { + builder.append("api.all.health"); + } + } + return builder.toString(); + } + + public void setContextData(Http.Request httpReq, org.sunbird.request.Request reqObj) { + try { + String reqContext = httpReq.attrs().get(Attrs.CONTEXT); + Map requestInfo = + objectMapper.readValue(reqContext, new TypeReference>() {}); + reqObj.setRequestId(httpReq.attrs().getOptional(Attrs.REQUEST_ID).orElse(null)); + reqObj.getContext().putAll((Map) requestInfo.get(JsonKey.CONTEXT)); + reqObj.getContext().putAll((Map) requestInfo.get(JsonKey.ADDITIONAL_INFO)); + } catch (Exception ex) { + ProjectCommonException.throwServerErrorException(ResponseCode.SERVER_ERROR); + } + } + + private void generateExceptionTelemetry(Request request, ProjectCommonException exception) { + try { + String reqContext = request.attrs().get(Attrs.CONTEXT); + Map requestInfo = objectMapper.readValue(reqContext, new TypeReference>() {}); + org.sunbird.request.Request reqForTelemetry = new org.sunbird.request.Request(); + Map params = (Map) requestInfo.getOrDefault(JsonKey.ADDITIONAL_INFO, new HashMap<>()); + params.put(JsonKey.LOG_TYPE, JsonKey.API_ACCESS); + params.put(JsonKey.MESSAGE, ""); + params.put(JsonKey.METHOD, request.method()); + params.put("err", exception.getResponseCode() + ""); + params.put("errtype", exception.getCode()); + long startTime = (Long) params.get(JsonKey.START_TIME); + params.put(JsonKey.DURATION, calculateApiTimeTaken(startTime)); + removeFields(params, JsonKey.START_TIME); + params.put(JsonKey.STATUS, String.valueOf(exception.getResponseCode())); + params.put(JsonKey.LOG_LEVEL, "error"); + params.put(JsonKey.STACKTRACE, generateStackTrace(exception.getStackTrace())); + reqForTelemetry.setRequest( + generateTelemetryRequestForController( + TelemetryEvents.ERROR.getName(), + params, + (Map) requestInfo.get(JsonKey.CONTEXT))); + TelemetryWriter.write(reqForTelemetry); + } catch (Exception ex) { + ex.printStackTrace(); + } + } +} diff --git a/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java b/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java new file mode 100644 index 00000000..739534e9 --- /dev/null +++ b/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java @@ -0,0 +1,55 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.keys.JsonKey; +import org.sunbird.request.Request; +import org.sunbird.response.ResponseCode; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Viewer resync API — recomputes a learner's collection roll-up from user_content_consumption. + * Idempotent (recompute-from-source), so safe to call repeatedly: backfill, drift repair, or a + * recompute after a collection is republished. Mirrors the legacy POST /v1/activity/agg, but targets + * the viewer aggregator (viewer-aggregator-actor, op "aggregate"). Runs on the userId-hashed pool, so + * a resync serialises with any live /v1/view/end for that learner — no race with real-time roll-ups. + * POST /v1/view/agg { request: { userId, collectionId|courseId, contextId|batchId } } + */ +public class ViewAggregateController extends BaseController { + + private final ActorRef viewerAggregatorActor; + + @Inject + public ViewAggregateController(@Named("viewer-aggregator-actor") ActorRef viewerAggregatorActor) { + this.viewerAggregatorActor = viewerAggregatorActor; + } + + public CompletionStage agg(Http.Request httpRequest) { + try { + Request request = createAndInitRequest("aggregate", httpRequest.body().asJson(), httpRequest); + validate(request); + return actorResponseHandler(viewerAggregatorActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + private void validate(Request request) { + String userId = (String) request.get(JsonKey.USER_ID); + Object courseId = request.get(JsonKey.COURSE_ID); + if (userId == null || userId.trim().isEmpty() + || courseId == null || courseId.toString().trim().isEmpty()) { + throw new ProjectCommonException( + ResponseCode.mandatoryParamsMissing.getErrorCode(), + "userId and courseId are mandatory", + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } +} diff --git a/modules/viewer/service/app/controllers/viewer/ViewController.java b/modules/viewer/service/app/controllers/viewer/ViewController.java new file mode 100644 index 00000000..72e0d159 --- /dev/null +++ b/modules/viewer/service/app/controllers/viewer/ViewController.java @@ -0,0 +1,104 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.commons.lang3.StringUtils; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.message.ResponseCode; +import org.sunbird.request.Request; +import play.mvc.Http; +import play.mvc.Result; + +import play.libs.Json; +import play.mvc.Results; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Granular view lifecycle APIs. Dispatches to ViewConsumptionActor (view-consumption-actor). + * POST /v1/view/start -> viewStart + * POST /v1/view/update -> viewUpdate + * POST /v1/view/end -> viewEnd + */ +public class ViewController extends BaseController { + + private final ActorRef viewConsumptionActor; + + @Inject + public ViewController(@Named("view-consumption-actor") ActorRef viewConsumptionActor) { + this.viewConsumptionActor = viewConsumptionActor; + } + + public CompletionStage viewStart(Http.Request httpRequest) { + return dispatch("viewStart", httpRequest); + } + + public CompletionStage viewUpdate(Http.Request httpRequest) { + return dispatch("viewUpdate", httpRequest); + } + + public CompletionStage viewEnd(Http.Request httpRequest) { + return dispatch("viewEnd", httpRequest); + } + + public CompletionStage viewRead(Http.Request httpRequest) { + return dispatch("viewRead", httpRequest); + } + + public CompletionStage assessmentSubmit(Http.Request httpRequest) { + return dispatch("viewAssess", httpRequest); + } + + public CompletionStage assessmentRead(Http.Request httpRequest) { + return dispatch("assessmentRead", httpRequest); + } + + public Result health(Http.Request httpRequest) { + ObjectNode json = Json.newObject(); + json.put("healthy", true); + return Results.ok(json); + } + + public Result preflight(String all) { + return Results.ok(); + } + + private CompletionStage dispatch(String operation, Http.Request httpRequest) { + try { + Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + validate(operation, request); + return actorResponseHandler(viewConsumptionActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + // Reject requests missing keys the actor unconditionally dereferences, so callers get a 400 with the + // offending field instead of an opaque 500/NPE. Contract is courseId/batchId/contentId (+ userId). + private void validate(String operation, Request request) { + switch (operation) { + case "viewStart": case "viewUpdate": case "viewEnd": case "viewAssess": case "assessmentRead": + requireNonBlank(request, "userId", "courseId", "batchId", "contentId"); + break; + case "viewRead": + requireNonBlank(request, "userId", "courseId", "batchId"); + break; + default: // no mandatory fields + } + } + + private void requireNonBlank(Request request, String... keys) { + for (String key : keys) { + if (StringUtils.isBlank((String) request.getRequest().get(key))) { + throw new ProjectCommonException( + ResponseCode.mandatoryParameterMissing.getErrorCode(), + ResponseCode.mandatoryParameterMissing.getErrorMessage() + " " + key, + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } + } +} diff --git a/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java b/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java new file mode 100644 index 00000000..1701f4e3 --- /dev/null +++ b/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java @@ -0,0 +1,85 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.commons.lang3.StringUtils; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.message.ResponseCode; +import org.sunbird.request.Request; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Summary APIs. Dispatches to ViewerSummaryActor (viewer-summary-actor). + * POST /v1/summary/read + * GET /v1/summary/list/:userId + * DELETE /v1/summary/delete/:userId + */ +public class ViewSummaryController extends BaseController { + + private final ActorRef viewerSummaryActor; + + @Inject + public ViewSummaryController(@Named("viewer-summary-actor") ActorRef viewerSummaryActor) { + this.viewerSummaryActor = viewerSummaryActor; + } + + public CompletionStage summaryRead(Http.Request httpRequest) { + return dispatchBody("summaryRead", httpRequest); + } + + public CompletionStage summaryList(String userId, Http.Request httpRequest) { + try { + Request request = createAndInitRequest("summaryList", httpRequest); + request.getRequest().put("userId", userId); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + public CompletionStage summaryDownload(String userId, Http.Request httpRequest) { + try { + Request request = createAndInitRequest("summaryDownload", httpRequest); + request.getRequest().put("userId", userId); + String[] fmt = httpRequest.queryString().getOrDefault("format", new String[]{"json"}); + request.getRequest().put("format", fmt.length > 0 ? fmt[0] : "json"); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + public CompletionStage summaryDelete(String userId, Http.Request httpRequest) { + try { + Request request = httpRequest.body().asJson() != null + ? createAndInitRequest("summaryDelete", httpRequest.body().asJson(), httpRequest) + : createAndInitRequest("summaryDelete", httpRequest); + request.getRequest().put("userId", userId); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + private CompletionStage dispatchBody(String operation, Http.Request httpRequest) { + try { + Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + // summaryRead identifies the enrolment by userId (from the body) — reject if absent. + if (StringUtils.isBlank((String) request.getRequest().get("userId"))) { + throw new ProjectCommonException( + ResponseCode.mandatoryParameterMissing.getErrorCode(), + ResponseCode.mandatoryParameterMissing.getErrorMessage() + " userId", + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } +} diff --git a/modules/viewer/service/app/filters/AccessLogFilter.java b/modules/viewer/service/app/filters/AccessLogFilter.java new file mode 100644 index 00000000..04cd23a4 --- /dev/null +++ b/modules/viewer/service/app/filters/AccessLogFilter.java @@ -0,0 +1,86 @@ +package filters; + +import org.apache.pekko.util.ByteString; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.telemetry.util.TelemetryEvents; +import org.sunbird.telemetry.util.TelemetryWriter; +import play.libs.streams.Accumulator; +import play.mvc.EssentialAction; +import play.mvc.EssentialFilter; +import play.mvc.Result; +import util.Attrs; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.concurrent.Executor; + +public class AccessLogFilter extends EssentialFilter { + + private final Executor executor; + private ObjectMapper objectMapper = new ObjectMapper(); + public LoggerUtil logger = new LoggerUtil(this.getClass()); + + @Inject + public AccessLogFilter(Executor executor) { + super(); + this.executor = executor; + } + + @Override + public EssentialAction apply(EssentialAction next) { + return EssentialAction.of( + request -> { + long startTime = System.currentTimeMillis(); + Accumulator accumulator = next.apply(request); + return accumulator.map( + result -> { + long endTime = System.currentTimeMillis(); + long requestTime = endTime - startTime; + try { + org.sunbird.request.Request req = new org.sunbird.request.Request(); + Map params = new WeakHashMap<>(); + params.put(JsonKey.URL, request.uri()); + params.put(JsonKey.METHOD, request.method()); + params.put(JsonKey.LOG_TYPE, JsonKey.API_ACCESS); + params.put(JsonKey.MESSAGE, ""); + params.put(JsonKey.METHOD, request.method()); + params.put(JsonKey.DURATION, requestTime); + params.put(JsonKey.STATUS, result.status()); + params.put(JsonKey.LOG_LEVEL, JsonKey.INFO); + String contextDetails = request.attrs().getOptional(Attrs.CONTEXT).orElse(""); + if(StringUtils.isNotBlank(contextDetails)) { + Map context = + objectMapper.readValue( + contextDetails, new TypeReference>() {}); + req.setRequest( + generateTelemetryRequestForController( + TelemetryEvents.LOG.getName(), + params, + (Map) context.get(JsonKey.CONTEXT))); + TelemetryWriter.write(req); + } + } catch (Exception ex) { + logger.error("AccessLogFilter:apply Exception in writing telemetry", ex); + } + return result; + }, + executor); + }); + } + + private Map generateTelemetryRequestForController( + String eventType, Map params, Map context) { + + Map map = new HashMap<>(); + map.put(JsonKey.TELEMETRY_EVENT_TYPE, eventType); + map.put(JsonKey.CONTEXT, context); + map.put(JsonKey.PARAMS, params); + return map; + } +} diff --git a/modules/viewer/service/app/filters/CustomGzipFilter.java b/modules/viewer/service/app/filters/CustomGzipFilter.java new file mode 100644 index 00000000..f04922dc --- /dev/null +++ b/modules/viewer/service/app/filters/CustomGzipFilter.java @@ -0,0 +1,67 @@ +package filters; + +import org.apache.pekko.stream.Materializer; +import org.apache.http.HttpHeaders; +import org.sunbird.keys.JsonKey; +import org.sunbird.common.ProjectUtil; +import org.sunbird.request.HeaderParam; +import play.filters.gzip.GzipFilter; +import play.filters.gzip.GzipFilterConfig; +import play.mvc.EssentialAction; +import play.mvc.EssentialFilter; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import java.util.function.BiFunction; + +public class CustomGzipFilter extends EssentialFilter { + private static boolean GzipFilterEnabled = + Boolean.parseBoolean(ProjectUtil.getConfigValue(JsonKey.SUNBIRD_GZIP_ENABLE)); + private static final double gzipThreshold = + Double.parseDouble(ProjectUtil.getConfigValue(JsonKey.SUNBIRD_GZIP_SIZE_THRESHOLD)); + private static final String GZIP = "gzip"; + // Size of buffer to use for gzip. + private static final int BUFFER_SIZE = 8192; + // Content length threshold, after which the filter will switch to chunking the result. + private static final int CHUNKED_THRESHOLD = 102400; + + private GzipFilter gzipFilter; + + @Inject + public CustomGzipFilter(Materializer materializer) { + GzipFilterConfig gzipFilterConfig = new GzipFilterConfig(); + gzipFilter = new GzipFilter( + gzipFilterConfig.withBufferSize(BUFFER_SIZE) + .withChunkedThreshold(CHUNKED_THRESHOLD) + .withShouldGzip((BiFunction) + (req, res) -> shouldGzipFunction(req, res)), + materializer + ); + } + + @Override + public EssentialAction apply(EssentialAction essentialAction) { + return gzipFilter.asJava().apply(essentialAction); + } + + // Whether the given request/result should be gzipped or not + private static boolean shouldGzipFunction(Http.RequestHeader requestHeader, Result responseHeader) { + double responseSize = 0.0; + boolean responseLengthKeyExist = responseHeader.headers().containsKey(HeaderParam.X_Response_Length.getName()); + if (responseLengthKeyExist) { + if (responseHeader.headers().get(HeaderParam.X_Response_Length.getName()) != null) { + String strValue = responseHeader.headers().get(HeaderParam.X_Response_Length.getName()); + responseSize = Double.parseDouble(strValue); + } + } + if (GzipFilterEnabled && (requestHeader.header(HttpHeaders.ACCEPT_ENCODING) != null)) { + if (requestHeader.header(HttpHeaders.ACCEPT_ENCODING).toString().toLowerCase().contains(GZIP)) { + if (responseSize >= gzipThreshold) { + return true; + } + } + } + return false; + } +} diff --git a/modules/viewer/service/app/filters/ResponseFilter.scala b/modules/viewer/service/app/filters/ResponseFilter.scala new file mode 100644 index 00000000..60469bad --- /dev/null +++ b/modules/viewer/service/app/filters/ResponseFilter.scala @@ -0,0 +1,40 @@ +package filters + +import org.apache.pekko.stream.Materializer +import org.apache.pekko.util.ByteString +import org.apache.commons.lang3.StringUtils +import org.sunbird.keys.JsonKey +import org.sunbird.keys.JsonKey.{CLOUD_STORAGE_CNAME_URL, CLOUD_STORE_BASE_PATH, CONTENT_CLOUD_STORAGE_CONTAINER} +import org.sunbird.common.ProjectUtil.getConfigValue +import play.api.Logging +import play.api.http.HttpEntity.Strict +import play.api.mvc.{Filter, RequestHeader, Result} + +import javax.inject.Inject +import scala.concurrent.{ExecutionContext, Future} + +class ResponseFilter @Inject()(implicit val mat: Materializer, ec: ExecutionContext) extends Filter with Logging { + + override def apply(nextFilter: (RequestHeader) => Future[Result])(rh: RequestHeader) = + nextFilter(rh) flatMap { result => + if (null != result.body && !result.body.isKnownEmpty){ + val contentType = result.body.contentType + val updatedBody = result.body.consumeData.map { x => + val y = x.utf8String.replaceAll(getConfigValue(JsonKey.CLOUD_STORE_BASE_PATH_PLACEHOLDER), getBaseUrl + "/" + getConfigValue(CONTENT_CLOUD_STORAGE_CONTAINER)) + logger.info("updated body: " + y) + y + } + updatedBody map { x => + result.copy(body = Strict(ByteString(x), contentType)) + } + } else { + Future(result) + } + } + + def getBaseUrl: String = { + var baseUrl = getConfigValue(CLOUD_STORAGE_CNAME_URL) + if (StringUtils.isEmpty(baseUrl)) baseUrl = getConfigValue(CLOUD_STORE_BASE_PATH) + baseUrl + } +} \ No newline at end of file diff --git a/modules/viewer/service/app/mapper/RequestMapper.java b/modules/viewer/service/app/mapper/RequestMapper.java new file mode 100644 index 00000000..7e391865 --- /dev/null +++ b/modules/viewer/service/app/mapper/RequestMapper.java @@ -0,0 +1,225 @@ +/** */ +package mapper; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.common.ProjectUtil; +import org.sunbird.request.Request; +import org.sunbird.response.ResponseCode; +import play.libs.Json; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * This class will map the requested json data into custom class. + * + * @author Manzarul + */ +public class RequestMapper { + public static LoggerUtil logger = new LoggerUtil(RequestMapper.class); + + /** + * Method to map request + * + * @param requestData JsonNode + * @param obj Class + * @exception RuntimeException + * @return + */ + public static Object mapRequest(JsonNode requestData, Class obj) throws Exception { + // First convert the JsonNode through our Scala collection conversion + Map convertedMap = mapRequest(requestData); + + // Now convert the cleaned map to the target object type + ObjectMapper mapper = new ObjectMapper(); + Object result = mapper.convertValue(convertedMap, obj); + + // For Request objects, ensure the internal request map also uses converted collections + if (result instanceof Request) { + Request requestObj = (Request) result; + Map internalRequest = requestObj.getRequest(); + if (internalRequest != null) { + Map convertedInternalRequest = new HashMap<>(); + for (Map.Entry entry : internalRequest.entrySet()) { + convertedInternalRequest.put(entry.getKey(), convertScalaCollections(entry.getValue())); + } + requestObj.setRequest(convertedInternalRequest); + } + } + + return result; + } + + /** + * Method to map request with Scala collection conversion + * + * @param requestData JsonNode + * @exception RuntimeException + * @return Map + */ + public static Map mapRequest(JsonNode requestData) throws Exception { + if (requestData == null) { + return new HashMap<>(); + } + + ObjectMapper mapper = new ObjectMapper(); + Map map = mapper.convertValue(requestData, new TypeReference>() {}); + + // Convert any Scala collections (Maps, Lists, etc.) to Java collections + Map convertedMap = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + Object convertedValue = convertScalaCollections(entry.getValue()); + convertedMap.put(entry.getKey(), convertedValue); + } + + return convertedMap; + } + + /** + * Helper method to convert Scala Map to Java Map recursively, including List conversions + * @param obj The Scala Map to convert + * @return Java Map + */ + private static Object convertToJavaMap(Object obj) { + if (obj instanceof Map && !(obj instanceof java.util.Map)) { + try { + // Convert Scala Map to Java Map + scala.collection.Map scalaMap = (scala.collection.Map) obj; + Map javaMap = new LinkedHashMap<>(); + + // Iterate through Scala map and convert each entry + scala.collection.Iterator iterator = scalaMap.iterator(); + while (iterator.hasNext()) { + scala.Tuple2 entry = (scala.Tuple2) iterator.next(); + Object key = entry._1(); + Object value = entry._2(); + + // Recursively convert nested maps and handle Scala collections + Object convertedValue = convertScalaCollections(value); + javaMap.put(key, convertedValue); + } + + return javaMap; + } catch (Exception e) { + logger.debug("Failed to convert Scala Map to Java Map: " + e.getMessage() + + ". Object type: " + obj.getClass().getName() + + ". Returning original object. Exception: " + e.toString()); + return obj; + } + } else { + return obj; + } + } + + /** + * Convert Scala collections to Java collections recursively + */ + private static Object convertScalaCollections(Object obj) { + if (obj == null) { + return null; + } + + // Handle Scala Lists (including :: cons lists) + if (obj instanceof scala.collection.Seq && !(obj instanceof java.util.List)) { + try { + scala.collection.Seq scalaSeq = (scala.collection.Seq) obj; + List javaList = new ArrayList<>(); + + scala.collection.Iterator iterator = scalaSeq.iterator(); + while (iterator.hasNext()) { + Object element = iterator.next(); + // Recursively convert nested collections + Object convertedElement = convertScalaCollections(element); + javaList.add(convertedElement); + } + + return javaList; + } catch (Exception e) { + logger.debug("Failed to convert Scala Seq to Java List: " + e.getMessage() + + ". Object type: " + obj.getClass().getName() + + ". Returning original object. Exception: " + e.toString()); + return obj; + } + } + + // Handle Scala Maps + if (obj instanceof scala.collection.Map && !(obj instanceof java.util.Map)) { + return convertToJavaMap(obj); + } + + // Handle Java Lists (process recursively for nested collections) + if (obj instanceof java.util.List) { + List javaList = (List) obj; + List convertedList = new ArrayList<>(); + for (Object element : javaList) { + convertedList.add(convertScalaCollections(element)); + } + return convertedList; + } + + // Handle Java Maps (process recursively for nested collections) + if (obj instanceof java.util.Map) { + Map javaMap = (Map) obj; + Map convertedMap = new LinkedHashMap<>(); + for (Map.Entry entry : javaMap.entrySet()) { + Object convertedValue = convertScalaCollections(entry.getValue()); + convertedMap.put(entry.getKey(), convertedValue); + } + return convertedMap; + } + + // Return other objects as-is + return obj; + } + + /** + * Helper method to convert Scala collections to Java collections + * @param value The value to potentially convert + * @param fieldName The field name for logging + * @return Java collection or original value + */ + private static Object convertScalaCollectionToJava(Object value, String fieldName) { + if (value == null) { + return value; + } + + String className = value.getClass().getName(); + + // Handle all Scala collection types + if (className.startsWith("scala.collection")) { + // Check if it's a Scala List (including :: cons lists) + if (className.contains("List") || className.contains("$colon$colon") || + className.contains("immutable.Nil") || value instanceof Iterable) { + List javaList = new ArrayList<>(); + + try { + // Handle Scala collections that are Iterable + if (value instanceof Iterable) { + for (Object item : (Iterable) value) { + // Recursively convert nested structures + if (item instanceof scala.collection.Map && !(item instanceof java.util.Map)) { + item = convertToJavaMap(item); + } else { + item = convertScalaCollectionToJava(item, fieldName + "[element]"); + } + javaList.add(item); + } + } + + return javaList; + + } catch (Exception e) { + logger.error("Failed to convert Scala collection for field " + fieldName + ": " + e.getMessage(), e); + return value; // Return original value if conversion fails + } + } + } + + return value; + } +} diff --git a/modules/viewer/service/app/modules/ActorStartModule.java b/modules/viewer/service/app/modules/ActorStartModule.java new file mode 100644 index 00000000..b0bd2624 --- /dev/null +++ b/modules/viewer/service/app/modules/ActorStartModule.java @@ -0,0 +1,43 @@ +package modules; + +import org.apache.pekko.routing.ConsistentHashingPool; +import org.apache.pekko.routing.ConsistentHashingRouter.ConsistentHashMapper; +import org.apache.pekko.routing.FromConfig; +import org.apache.pekko.routing.RouterConfig; +import com.google.inject.AbstractModule; +import play.libs.pekko.PekkoGuiceSupport; +import util.ACTOR_NAMES; + +public class ActorStartModule extends AbstractModule implements PekkoGuiceSupport { + + @Override + protected void configure() { + System.out.println("binding actors for dependency injection"); + final RouterConfig config = new FromConfig(); + + // viewer-aggregator: PROGRAMMATIC ConsistentHashingPool keyed on userId — serialises one user's + // roll-ups. A config-only consistent-hashing router has no hash key for plain Request messages + // and would send them to deadLetters. + final ConsistentHashMapper userIdHashMapper = + message -> { + if (message instanceof org.sunbird.request.Request) { + Object uid = ((org.sunbird.request.Request) message).get("userId"); + return uid != null ? uid : ""; + } + return ""; + }; + + for (ACTOR_NAMES actor : ACTOR_NAMES.values()) { + if (ACTOR_NAMES.VIEWER_AGGREGATOR_ACTOR.equals(actor)) { + bindActor( + actor.getActorClass(), + actor.getActorName(), + props -> props.withRouter(new ConsistentHashingPool(8).withHashMapper(userIdHashMapper)) + .withDispatcher("pekko.actor.viewer-dispatcher")); + } else { + bindActor(actor.getActorClass(), actor.getActorName(), props -> props.withRouter(config)); + } + } + System.out.println("binding completed"); + } +} diff --git a/modules/viewer/service/app/modules/ApplicationStart.java b/modules/viewer/service/app/modules/ApplicationStart.java new file mode 100644 index 00000000..5e998ee1 --- /dev/null +++ b/modules/viewer/service/app/modules/ApplicationStart.java @@ -0,0 +1,83 @@ +package modules; + +import org.sunbird.auth.verifier.KeyManager; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.common.ProjectUtil; +import org.sunbird.learner.util.ContentSearchMock; +import org.sunbird.learner.util.SchedulerManager; +import org.sunbird.learner.util.Util; +import play.api.Environment; +import play.api.inject.ApplicationLifecycle; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.concurrent.CompletableFuture; + +/** + * This class will be called after on application startup. only one instance of this class will be + * created. StartModule class has responsibility to eager load this class. + * + * @author Jaikumar Soundara Rajan + */ +@Singleton +public class ApplicationStart { + + public static ProjectUtil.Environment env; + public static String ssoPublicKey = ""; + public LoggerUtil logger = new LoggerUtil(this.getClass()); + + /** + * All one time initialization which required during server startup will fall here. + * + * @param lifecycle ApplicationLifecycle + * @param environment Environment + */ + @Inject + public ApplicationStart(ApplicationLifecycle lifecycle, Environment environment) { + System.out.println("ApplicationStart:ApplicationStart: Start"); + setEnvironment(environment); + ssoPublicKey = System.getenv(JsonKey.SSO_PUBLIC_KEY); + logger.info("Server started.. with environment: " + env.name()); + if (Boolean.parseBoolean(ProjectUtil.getConfigValue(JsonKey.CONTENT_SERVICE_MOCK_ENABLED))) { + mockServiceSetup(); + } + checkCassandraConnections(); + SchedulerManager.schedule(); + lifecycle.addStopHook( + () -> { + return CompletableFuture.completedFuture(null); + }); + System.out.println("keymanger.init():starts"); + KeyManager.init(); + System.out.println("ApplicationStart:ApplicationStart: End"); + } + + public static void mockServiceSetup() { + LoggerUtil logger = new LoggerUtil(ApplicationStart.class); + try { + ContentSearchMock.setup(); + } catch (Exception e) { + logger.info((org.sunbird.request.RequestContext) null,"Error setting up ContentSearchMock:"+e); + } + } + + private void checkCassandraConnections() { + Util.checkCassandraDbConnections(); + } + + /** + * This method will identify the environment and update with enum. + * + * @return Environment + */ + public ProjectUtil.Environment setEnvironment(Environment environment) { + if (environment.asJava().isDev()) { + return env = ProjectUtil.Environment.dev; + } else if (environment.asJava().isTest()) { + return env = ProjectUtil.Environment.qa; + } else { + return env = ProjectUtil.Environment.prod; + } + } +} diff --git a/modules/viewer/service/app/modules/ErrorHandler.java b/modules/viewer/service/app/modules/ErrorHandler.java new file mode 100644 index 00000000..a275fc61 --- /dev/null +++ b/modules/viewer/service/app/modules/ErrorHandler.java @@ -0,0 +1,61 @@ +package modules; + +import com.typesafe.config.Config; +import controllers.BaseController; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.response.Response; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.response.ResponseCode; +import play.Environment; +import play.api.OptionalSourceMapper; +import play.api.routing.Router; +import play.http.DefaultHttpErrorHandler; +import play.libs.Json; +import play.mvc.Http; +import play.mvc.Result; +import play.mvc.Results; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +@Singleton +public class ErrorHandler extends DefaultHttpErrorHandler { + public LoggerUtil logger = new LoggerUtil(this.getClass()); + + @Inject + public ErrorHandler( + Config config, + Environment environment, + OptionalSourceMapper sourceMapper, + Provider routes) { + super(config, environment, sourceMapper, routes); + } + + @Override + public CompletionStage onServerError(Http.RequestHeader request, Throwable t) { + logger.error("Global: onError called for path = " + request.path(), t); + Response response = null; + ProjectCommonException commonException = null; + if (t instanceof ProjectCommonException) { + commonException = (ProjectCommonException) t; + } else if (t instanceof org.apache.pekko.pattern.AskTimeoutException) { + commonException = + new ProjectCommonException( + ResponseCode.actorConnectionError.getErrorCode(), + ResponseCode.actorConnectionError.getErrorMessage(), + ResponseCode.SERVER_ERROR.getResponseCode()); + } else { + commonException = + new ProjectCommonException( + ResponseCode.internalError.getErrorCode(), + ResponseCode.internalError.getErrorMessage(), + ResponseCode.SERVER_ERROR.getResponseCode()); + } + response = + BaseController.createResponseOnException(request.path(), request.method(), commonException); + return CompletableFuture.completedFuture(Results.internalServerError(Json.toJson(response))); + } +} diff --git a/modules/viewer/service/app/modules/OnRequestHandler.java b/modules/viewer/service/app/modules/OnRequestHandler.java new file mode 100644 index 00000000..d77b6acf --- /dev/null +++ b/modules/viewer/service/app/modules/OnRequestHandler.java @@ -0,0 +1,283 @@ +package modules; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.typesafe.config.ConfigFactory; +import controllers.BaseController; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.auth.verifier.AccessTokenValidator; +import org.sunbird.cache.platform.Platform; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.response.Response; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.common.ProjectUtil; +import org.sunbird.request.HeaderParam; +import org.sunbird.response.ResponseCode; +import org.sunbird.utils.JsonUtil; +import play.http.ActionCreator; +import play.libs.Json; +import play.mvc.Action; +import play.mvc.Http; +import play.mvc.Result; +import play.mvc.Results; +import util.Attrs; +import util.RequestInterceptor; + +import java.lang.reflect.Method; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.WeakHashMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.stream.Collectors; + +public class OnRequestHandler implements ActionCreator { + + private ObjectMapper mapper = new ObjectMapper(); + public static boolean isServiceHealthy = true; + private final List USER_UNAUTH_STATES = + Arrays.asList(JsonKey.UNAUTHORIZED, JsonKey.ANONYMOUS); + public LoggerUtil logger = new LoggerUtil(this.getClass()); + private static final List clientAppHeaderKeys = Platform.getStringList("request_headers_logging", Arrays.asList("x-app-id", "x-device-id", "x-channel-id")); + + + @Override + public Action createAction(Http.Request request, Method actionMethod) { + Optional optionalMessageId = request.header(JsonKey.MESSAGE_ID); + String messageId; + if (optionalMessageId.isPresent()) { + messageId = optionalMessageId.get(); + } else { + UUID uuid = UUID.randomUUID(); + messageId = uuid.toString(); + } + return new Action.Simple() { + @Override + public CompletionStage call(Http.Request request) { + CompletionStage result = checkForServiceHealth(request); + String message = null; + if (result != null) { + return result; + } + if (ConfigFactory.load().getBoolean(JsonKey.AUTH_ENABLED)) { + message = RequestInterceptor.verifyRequestData(request); + } else { + message = JsonKey.ANONYMOUS; + } + // Setting Actual userId (requestedBy) and managed userId (requestedFor) placeholders in flash memory to null before processing. + // Unauthorized, Anonymous, UserID + Optional forAuth = request.header(HeaderParam.X_Authenticated_For.getName()); + String childId = null; + String loggingHeaders = getLoggingHeaders(request); + request = request.addAttr(Attrs.X_LOGGING_HEADERS, loggingHeaders); + if (StringUtils.isNotBlank(message) && forAuth.isPresent() && StringUtils.isNotBlank(forAuth.orElse(""))) { + String requestedForId = getRequestedForId(request); + childId = AccessTokenValidator.verifyManagedUserToken(forAuth.get(), message, requestedForId, loggingHeaders); + if (StringUtils.isNotBlank(childId) && !USER_UNAUTH_STATES.contains(childId)) { + request = request.addAttr(Attrs.REQUESTED_FOR, childId); + } + + } + // call method to set all the required params for the telemetry event(log)... + request = intializeRequestInfo(request, message, messageId); + request = request.addAttr(Attrs.X_AUTH_TOKEN, request.header(HeaderParam.X_Authenticated_User_Token.getName()).orElse("")); + if ((!USER_UNAUTH_STATES.contains(message)) && (childId==null || !USER_UNAUTH_STATES.contains(childId))) { + request = request.addAttr(Attrs.USER_ID, message); + request = request.addAttr(Attrs.IS_AUTH_REQ, "false"); + for (String uri : RequestInterceptor.restrictedUriList) { + if (request.path().contains(uri)) { + request = request.addAttr(Attrs.IS_AUTH_REQ, "true"); + break; + } + } + result = delegate.call(request); + } else if (JsonKey.UNAUTHORIZED.equals(message) || (childId != null && JsonKey.UNAUTHORIZED.equals(childId))) { + String errorCode = JsonKey.UNAUTHORIZED.equals(message) ? message : childId; + result = onDataValidationError(request, errorCode, ResponseCode.UNAUTHORIZED.getResponseCode()); + } else { + result = delegate.call(request); + } + return result.thenApply(res -> res.withHeader("Access-Control-Allow-Origin", "*")); + } + }; + } + + private String getRequestedForId(Http.Request request) { + String requestedForUserID = null; + JsonNode jsonBody = request.body().asJson(); + if(null != jsonBody && jsonBody.has(JsonKey.REQUEST) && jsonBody.get(JsonKey.REQUEST).has(JsonKey.USER_ID)) { + requestedForUserID = jsonBody.get(JsonKey.REQUEST).get(JsonKey.USER_ID).asText(); + } else { // for read-api + String uuidSegment = null; + Path path = Paths.get(request.uri()); + if (request.queryString().isEmpty()) { + uuidSegment = path.getFileName().toString(); + } else { + String[] queryPath = path.getFileName().toString().split("\\?"); + uuidSegment = queryPath[0]; + } + try { + requestedForUserID = UUID.fromString(uuidSegment).toString(); + } catch (IllegalArgumentException iae) { + logger.info("Perhaps this is another API, like search that doesn't carry user id."); + } + } + return requestedForUserID; + } + + /** + * This method will do request data validation for GET method only. As a GET request user must + * send some key in header. + * + * @param request Request + * @param errorMessage String + * @return Promise + */ + public CompletionStage onDataValidationError( + Http.Request request, String errorMessage, int responseCode) { + logger.info("Data error found--" + errorMessage); + ResponseCode code = ResponseCode.getResponse(errorMessage); + ResponseCode headerCode = ResponseCode.CLIENT_ERROR; + Response resp = BaseController.createFailureResponse(request, code, headerCode); + return CompletableFuture.completedFuture(Results.status(responseCode, Json.toJson(resp))); + } + + private Http.Request intializeRequestInfo(Http.Request request, String userId, String requestId) { + try { + String actionMethod = request.method(); + String url = request.uri(); + String methodName = actionMethod; + long startTime = System.currentTimeMillis(); + String signType = ""; + String source = ""; + if (request.body() != null && request.body().asJson() != null) { + JsonNode requestNode = + request.body().asJson().get("params"); // extracting signup type from request + if (requestNode != null && requestNode.get(JsonKey.SIGNUP_TYPE) != null) { + signType = requestNode.get(JsonKey.SIGNUP_TYPE).asText(); + } + if (requestNode != null && requestNode.get(JsonKey.REQUEST_SOURCE) != null) { + source = requestNode.get(JsonKey.REQUEST_SOURCE).asText(); + } + } + Map reqContext = new WeakHashMap<>(); + request = request.addAttr(Attrs.SIGNUP_TYPE, signType); + reqContext.put(JsonKey.SIGNUP_TYPE, signType); + request = request.addAttr(Attrs.REQUEST_SOURCE, source); + reqContext.put(JsonKey.REQUEST_SOURCE, source); + + // set env and channel to the + Optional optionalChannel = request.header(HeaderParam.CHANNEL_ID.getName()); + String channel; + if (optionalChannel.isPresent()) { + channel = optionalChannel.get(); + } else { + String sunbirdDefaultChannel = ProjectUtil.getConfigValue(JsonKey.SUNBIRD_DEFAULT_CHANNEL); + channel = + (StringUtils.isNotEmpty(sunbirdDefaultChannel)) + ? sunbirdDefaultChannel + : JsonKey.DEFAULT_ROOT_ORG_ID; + } + reqContext.put(JsonKey.CHANNEL, channel); + request = request.addAttr(Attrs.CHANNEL, channel); + reqContext.put(JsonKey.ENV, getEnv(request)); + reqContext.put(JsonKey.REQUEST_ID, requestId); + Optional optionalAppId = request.header(HeaderParam.X_APP_ID.getName()); + // check if in request header X-app-id is coming then that need to + // be pass in search telemetry. + if (optionalAppId.isPresent()) { + request = request.addAttr(Attrs.APP_ID, optionalAppId.get()); + reqContext.put(JsonKey.APP_ID, optionalAppId.get()); + } + // checking device id in headers + Optional optionalDeviceId = request.header(HeaderParam.X_Device_ID.getName()); + if (optionalDeviceId.isPresent()) { + request = request.addAttr(Attrs.DEVICE_ID, optionalDeviceId.get()); + reqContext.put(JsonKey.DEVICE_ID, optionalDeviceId.get()); + } + if (!USER_UNAUTH_STATES.contains(userId)) { + reqContext.put(JsonKey.ACTOR_ID, userId); + reqContext.put(JsonKey.ACTOR_TYPE, StringUtils.capitalize(JsonKey.USER)); + request = request.addAttr(Attrs.ACTOR_ID, userId); + request = request.addAttr(Attrs.ACTOR_TYPE, JsonKey.USER); + } else { + Optional optionalConsumerId = request.header(HeaderParam.X_Consumer_ID.getName()); + String consumerId; + if (optionalConsumerId.isPresent()) { + consumerId = optionalConsumerId.get(); + } else { + consumerId = JsonKey.DEFAULT_CONSUMER_ID; + } + reqContext.put(JsonKey.ACTOR_ID, consumerId); + reqContext.put(JsonKey.ACTOR_TYPE, StringUtils.capitalize(JsonKey.CONSUMER)); + request = request.addAttr(Attrs.ACTOR_ID, consumerId); + request = request.addAttr(Attrs.ACTOR_TYPE, JsonKey.CONSUMER); + } + Map map = new WeakHashMap<>(); + map.put(JsonKey.CONTEXT, reqContext); + Map additionalInfo = new WeakHashMap<>(); + additionalInfo.put(JsonKey.URL, url); + additionalInfo.put(JsonKey.METHOD, methodName); + additionalInfo.put(JsonKey.START_TIME, startTime); + + // additional info contains info other than context info ... + map.put(JsonKey.ADDITIONAL_INFO, additionalInfo); + if (StringUtils.isBlank(requestId)) { + requestId = JsonKey.DEFAULT_CONSUMER_ID; + } + request = request.addAttr(Attrs.REQUEST_ID, requestId); + request = request.addAttr(Attrs.CONTEXT, mapper.writeValueAsString(map)); + } catch (Exception e) { + ProjectCommonException.throwServerErrorException(ResponseCode.SERVER_ERROR, e.getMessage()); + } + return request; + } + + private String getEnv(Http.Request request) { + + String uri = request.uri(); + String env; + if (uri.startsWith("/v1/page")) { + env = JsonKey.PAGE; + } else if (uri.startsWith("/v1/course/batch")) { + env = JsonKey.BATCH; + } else if (uri.startsWith("/v1/dashboard")) { + env = JsonKey.DASHBOARD; + } else if (uri.startsWith("/v1/content")) { + env = JsonKey.BATCH; + } else { + env = "miscellaneous"; + } + return env; + } + + public CompletionStage checkForServiceHealth(Http.Request request) { + if (Boolean.parseBoolean((ProjectUtil.getConfigValue(JsonKey.SUNBIRD_HEALTH_CHECK_ENABLE))) + && !request.path().endsWith(JsonKey.HEALTH)) { + if (!isServiceHealthy) { + ResponseCode headerCode = ResponseCode.SERVICE_UNAVAILABLE; + Response resp = BaseController.createFailureResponse(request, headerCode, headerCode); + return CompletableFuture.completedFuture( + Results.status(ResponseCode.SERVICE_UNAVAILABLE.getResponseCode(), Json.toJson(resp))); + } + } + return null; + } + + // TODO: same method created in BaseController also. We should move it to a common place. + protected String getLoggingHeaders(Http.Request httpRequest) { + try { + Map> headers = httpRequest.getHeaders().toMap(); + Map> filteredHeaders = headers.entrySet().stream().filter(e -> clientAppHeaderKeys.contains(e.getKey().toLowerCase())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return JsonUtil.serialize(filteredHeaders); + } catch (Exception e) { + return "Exception in serializing headers= " + e.getMessage(); + } + } +} diff --git a/modules/viewer/service/app/modules/StartModule.java b/modules/viewer/service/app/modules/StartModule.java new file mode 100644 index 00000000..3f9ba7d8 --- /dev/null +++ b/modules/viewer/service/app/modules/StartModule.java @@ -0,0 +1,29 @@ +package modules; + +import com.google.inject.AbstractModule; +import org.sunbird.cache.util.RedisCacheUtil; +import org.sunbird.common.ProjectUtil; + +/** + * This class is responsible for creating instance of + * ApplicationStart at server startup time. + * + * @author Jaikumar Soundara Rajan + */ +public class StartModule extends AbstractModule { + @Override + protected void configure() { + System.out.println("StartModule:configure: Start"); + try { + bind(ApplicationStart.class).asEagerSingleton(); + if (Boolean.parseBoolean(ProjectUtil.getConfigValue("redis.enabled"))) { + bind(RedisCacheUtil.class).asEagerSingleton(); + } + } catch (Exception | Error e) { + e.printStackTrace(); + throw e; + } + System.out.println("StartModule:configure: End"); + + } +} diff --git a/modules/viewer/service/app/util/ACTOR_NAMES.java b/modules/viewer/service/app/util/ACTOR_NAMES.java new file mode 100644 index 00000000..c9bdc520 --- /dev/null +++ b/modules/viewer/service/app/util/ACTOR_NAMES.java @@ -0,0 +1,27 @@ +package util; + +import org.sunbird.viewer.actor.ViewConsumptionActor; +import org.sunbird.viewer.actor.ViewerAggregatorActor; +import org.sunbird.viewer.actor.ViewerSummaryActor; + +public enum ACTOR_NAMES { + VIEW_CONSUMPTION_ACTOR(ViewConsumptionActor.class, "view-consumption-actor"), + VIEWER_AGGREGATOR_ACTOR(ViewerAggregatorActor.class, "viewer-aggregator-actor"), + VIEWER_SUMMARY_ACTOR(ViewerSummaryActor.class, "viewer-summary-actor"); + + private ACTOR_NAMES(Class clazz, String name) { + actorClass = clazz; + actorName = name; + } + + private Class actorClass; + private String actorName; + + public Class getActorClass() { + return actorClass; + } + + public String getActorName() { + return actorName; + } +} diff --git a/modules/viewer/service/app/util/Attrs.java b/modules/viewer/service/app/util/Attrs.java new file mode 100644 index 00000000..dac279a3 --- /dev/null +++ b/modules/viewer/service/app/util/Attrs.java @@ -0,0 +1,22 @@ +package util; + +import org.sunbird.keys.JsonKey; +import play.libs.typedmap.TypedKey; + +public class Attrs { + public static final TypedKey USER_ID = TypedKey.create(JsonKey.USER_ID); + public static final TypedKey AUTH_WITH_MASTER_KEY = TypedKey.create(JsonKey.AUTH_WITH_MASTER_KEY); + public static final TypedKey REQUEST_ID = TypedKey.create(JsonKey.REQUEST_ID); + public static final TypedKey CONTEXT = TypedKey.create(JsonKey.CONTEXT); + public static final TypedKey REQUESTED_FOR = TypedKey.create(JsonKey.REQUESTED_FOR); + public static final TypedKey IS_AUTH_REQ = TypedKey.create(JsonKey.IS_AUTH_REQ); + public static final TypedKey SIGNUP_TYPE = TypedKey.create(JsonKey.SIGNUP_TYPE); + public static final TypedKey REQUEST_SOURCE = TypedKey.create(JsonKey.REQUEST_SOURCE); + public static final TypedKey CHANNEL = TypedKey.create(JsonKey.CHANNEL); + public static final TypedKey APP_ID = TypedKey.create(JsonKey.APP_ID); + public static final TypedKey DEVICE_ID = TypedKey.create(JsonKey.DEVICE_ID); + public static final TypedKey ACTOR_ID = TypedKey.create(JsonKey.ACTOR_ID); + public static final TypedKey ACTOR_TYPE = TypedKey.create(JsonKey.ACTOR_TYPE); + public static final TypedKey X_AUTH_TOKEN = TypedKey.create(JsonKey.X_AUTH_TOKEN); + public static final TypedKey X_LOGGING_HEADERS = TypedKey.create(JsonKey.X_LOGGING_HEADERS); +} diff --git a/modules/viewer/service/app/util/AuthenticationHelper.java b/modules/viewer/service/app/util/AuthenticationHelper.java new file mode 100644 index 00000000..5249a992 --- /dev/null +++ b/modules/viewer/service/app/util/AuthenticationHelper.java @@ -0,0 +1,81 @@ +package util; + +import org.sunbird.auth.verifier.Base64Util; +import org.sunbird.cassandra.CassandraOperation; +import org.sunbird.response.Response; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.helper.ServiceFactory; +import org.sunbird.learner.util.Util; +import org.sunbird.learner.util.Util.DbInfo; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class will handle all the method related to authentication. For example verifying user + * access token, creating access token after success login. + * + * @author Manzarul + */ +public class AuthenticationHelper { + + private static CassandraOperation cassandraOperation = ServiceFactory.getInstance(); + private static DbInfo userAuth = Util.dbInfoMap.get(JsonKey.USER_AUTH_DB); + public static LoggerUtil logger = new LoggerUtil(AuthenticationHelper.class); + + /** + * This method will verify the incoming user access token against store data base /cache. If token + * is valid then it would be associated with some user id. In case of token matched it will + * provide user id. else will provide empty string. + * + * @param token String + * @return String + */ + @SuppressWarnings("unchecked") + public static String verifyUserAccessToken(String token) { + String userId = JsonKey.UNAUTHORIZED; + try { + Response authResponse = + cassandraOperation.getRecordByIdentifier(userAuth.getKeySpace(), userAuth.getTableName(), token, null, null); + if (authResponse != null && authResponse.get(JsonKey.RESPONSE) != null) { + List> authList = + (List>) authResponse.get(JsonKey.RESPONSE); + if (authList != null && !authList.isEmpty()) { + Map authMap = authList.get(0); + userId = (String) authMap.get(JsonKey.USER_ID); + } + } + } catch (Exception e) { + logger.error("invalid auth token =" + token, e); + } + return userId; + } + + @SuppressWarnings("unchecked") + public static String verifyClientAccessToken(String clientId, String clientToken) { + Util.DbInfo clientDbInfo = Util.dbInfoMap.get(JsonKey.CLIENT_INFO_DB); + Map propertyMap = new HashMap<>(); + propertyMap.put(JsonKey.ID, clientId); + propertyMap.put(JsonKey.MASTER_KEY, clientToken); + String validClientId = JsonKey.UNAUTHORIZED; + try { + Response clientResponse = + cassandraOperation.getRecordsByProperties( + clientDbInfo.getKeySpace(), clientDbInfo.getTableName(), propertyMap, null); + if (null != clientResponse && !clientResponse.getResult().isEmpty()) { + List> dataList = + (List>) clientResponse.getResult().get(JsonKey.RESPONSE); + validClientId = (String) dataList.get(0).get(JsonKey.ID); + } + } catch (Exception e) { + logger.error("Validating client token failed due to : ", e); + } + return validClientId; + } + + private static byte[] decodeFromBase64(String data) { + return Base64Util.decode(data, 11); + } +} diff --git a/modules/viewer/service/app/util/Common.java b/modules/viewer/service/app/util/Common.java new file mode 100644 index 00000000..753902d7 --- /dev/null +++ b/modules/viewer/service/app/util/Common.java @@ -0,0 +1,16 @@ +package util; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Common { + + public static Map getRequestHeadersInArray(Map> requestHeaders) { + Map requestHeadersArray = new HashMap(); + requestHeaders.entrySet().forEach(entry -> { + requestHeadersArray.put(entry.getKey(), (String[]) entry.getValue().toArray(new String[0])); + }); + return requestHeadersArray; + } +} diff --git a/modules/viewer/service/app/util/RequestInterceptor.java b/modules/viewer/service/app/util/RequestInterceptor.java new file mode 100644 index 00000000..ac69db4a --- /dev/null +++ b/modules/viewer/service/app/util/RequestInterceptor.java @@ -0,0 +1,145 @@ +package util; + +import org.apache.commons.lang3.StringUtils; +import org.sunbird.auth.verifier.AccessTokenValidator; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.request.HeaderParam; +import play.mvc.Http; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Request interceptor responsible to authenticated HTTP requests + * + * @author Amit Kumar + */ +public class RequestInterceptor { + + public static List restrictedUriList = null; + private static ConcurrentHashMap apiHeaderIgnoreMap = new ConcurrentHashMap<>(); + private static LoggerUtil logger = new LoggerUtil(RequestInterceptor.class); + + private RequestInterceptor() {} + + static { + restrictedUriList = new ArrayList<>(); + restrictedUriList.add("/v1/content/state/update"); + + short var = 1; + apiHeaderIgnoreMap.put("/service/health", var); + apiHeaderIgnoreMap.put("/v1/page/assemble", var); + apiHeaderIgnoreMap.put("/v1/dial/assemble", var); + apiHeaderIgnoreMap.put("/health", var); + apiHeaderIgnoreMap.put("/v1/data/sync", var); + apiHeaderIgnoreMap.put("/v1/content/link", var); + apiHeaderIgnoreMap.put("/v1/content/unlink", var); + apiHeaderIgnoreMap.put("/v1/content/link/search", var); + apiHeaderIgnoreMap.put("/v1/course/batch/search", var); + apiHeaderIgnoreMap.put("/v1/cache/clear", var); + apiHeaderIgnoreMap.put("/private/v1/course/batch/create", var); + apiHeaderIgnoreMap.put("/v1/course/create", var); + apiHeaderIgnoreMap.put("/v2/user/courses/list", var); + apiHeaderIgnoreMap.put("/v1/collection/summary", var); + } + + /** + * Authenticates given HTTP request context + * + * @param request HTTP play request + * @return User or Client ID for authenticated request. For unauthenticated requests, UNAUTHORIZED + * is returned + */ + public static String verifyRequestData(Http.Request request) { + String clientId = JsonKey.UNAUTHORIZED; + Optional accessToken = request.header(HeaderParam.X_Authenticated_User_Token.getName()); + Optional authClientToken = + request.header(HeaderParam.X_Authenticated_Client_Token.getName()); + Optional authClientId = request.header(HeaderParam.X_Authenticated_Client_Id.getName()); + if (!isRequestInExcludeList(request.path()) && !isRequestPrivate(request.path())) { + if (accessToken.isPresent()) { + // This is to handle Mobile App expired token for content state update API. + if (StringUtils.contains(request.path(), "v1/content/state/update")) { + clientId = AccessTokenValidator.verifyUserToken(accessToken.get(), false); + } else { + clientId = AccessTokenValidator.verifyUserToken(accessToken.get(), true); + } + } else if (authClientToken.isPresent() && authClientId.isPresent()) { + clientId = + AuthenticationHelper.verifyClientAccessToken(authClientId.get(), authClientToken.get()); + if (!JsonKey.UNAUTHORIZED.equals(clientId)) { + request = request.addAttr(Attrs.AUTH_WITH_MASTER_KEY, Boolean.toString(true)); + } + } + return clientId; + } else { + if (accessToken.isPresent()) { + String clientAccessTokenId = null; + try { + // This is to handle Mobile App expired token for content state update API. + if (StringUtils.contains(request.path(), "v1/content/state/update")) { + clientAccessTokenId = AccessTokenValidator.verifyUserToken(accessToken.get(), false); + } else { + clientAccessTokenId = AccessTokenValidator.verifyUserToken(accessToken.get(), true); + } + if (JsonKey.UNAUTHORIZED.equalsIgnoreCase(clientAccessTokenId)) { + clientAccessTokenId = null; + } + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + clientAccessTokenId = null; + } + return StringUtils.isNotBlank(clientAccessTokenId) + ? clientAccessTokenId + : JsonKey.ANONYMOUS; + } + return JsonKey.ANONYMOUS; + } + } + + private static boolean isRequestPrivate(String path) { + return path.contains(JsonKey.PRIVATE); + } + + /** + * Checks if request URL is in excluded (i.e. public) URL list or not + * + * @param requestUrl Request URL + * @return True if URL is in excluded (public) URLs. Otherwise, returns false + */ + public static boolean isRequestInExcludeList(String requestUrl) { + boolean resp = false; + if (!StringUtils.isBlank(requestUrl)) { + if (apiHeaderIgnoreMap.containsKey(requestUrl)) { + resp = true; + } else { + String[] splitPath = requestUrl.split("[/]"); + String urlWithoutPathParam = removeLastValue(splitPath); + if (apiHeaderIgnoreMap.containsKey(urlWithoutPathParam)) { + resp = true; + } + } + } + return resp; + } + + /** + * Returns URL without path and query parameters. + * + * @param splitPath URL path split on slash (i.e. /) + * @return URL without path and query parameters + */ + private static String removeLastValue(String splitPath[]) { + + StringBuilder builder = new StringBuilder(); + if (splitPath != null && splitPath.length > 0) { + for (int i = 1; i < splitPath.length - 1; i++) { + builder.append("/" + splitPath[i]); + } + } + return builder.toString(); + } +} diff --git a/modules/viewer/service/conf/application.conf b/modules/viewer/service/conf/application.conf new file mode 100644 index 00000000..a9e48cec --- /dev/null +++ b/modules/viewer/service/conf/application.conf @@ -0,0 +1,404 @@ +# This is the main configuration file for the application. +# https://www.playframework.com/documentation/latest/ConfigFile +# ~~~~~ + +## Pekko +# https://www.playframework.com/documentation/latest/JavaPekko#Configuration +# ~~~~~ +pekko { + loggers = ["org.apache.pekko.event.slf4j.Slf4jLogger"] + loglevel = "INFO" + stdout-loglevel = "DEBUG" + logging-filter = "org.apache.pekko.event.slf4j.Slf4jLoggingFilter" + log-config-on-start = off + + actor { + provider = "org.apache.pekko.actor.LocalActorRefProvider" + serializers { + java = "org.apache.pekko.serialization.JavaSerializer" + } + serialization-bindings { + "org.sunbird.request.Request" = java + "org.sunbird.response.Response" = java + } + default-dispatcher { + # This will be used if you have set "executor = "fork-join-executor"" + fork-join-executor { + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 32.0 + + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 64 + + # Setting to "FIFO" to use queue like peeking mode which "poll" or "LIFO" to use stack + # like peeking mode which "pop". + task-peeking-mode = "FIFO" + } + } + page-mgr-actor-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + tracking-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + rr-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + brr-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 1 + parallelism-factor = 2.0 + parallelism-max = 4 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + deployment { + /view-consumption-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = tracking-dispatcher + } + # viewer-aggregator-actor is bound programmatically in ActorStartModule with a + # ConsistentHashingPool + hashMapper keyed on userId (per-user serialization). + # No deployment entry here: config FromConfig cannot supply a hash key. + /viewer-summary-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /page-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 10 + dispatcher = page-mgr-actor-dispatcher + } + /cache-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /course-metrics-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /course-enrolment-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = tracking-dispatcher + } + /content-consumption-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = tracking-dispatcher + } + /course-batch-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /search-handler-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /health-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /course-batch-certificate-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /certificate-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /qrcode-download-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /bulk-upload-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /es-sync-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /bulk-upload-background-job-actor + { + router = smallest-mailbox-pool + nr-of-instances = 2 + dispatcher = brr-dispatcher + } + /course-batch-notification-actor + { + router = smallest-mailbox-pool + nr-of-instances = 2 + dispatcher = brr-dispatcher + } + /background-job-manager-actor + { + router = smallest-mailbox-pool + nr-of-instances = 2 + dispatcher = brr-dispatcher + } + /course-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /group-aggregates-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /collection-summary-aggregate-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /exhaust-job-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /assessment-aggregator-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /activity-aggregator-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /observability-report-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + } + } +} + +## Internationalisation +# https://www.playframework.com/documentation/latest/JavaI18N +# ~~~~~ +play.i18n { + # The application languages + langs = [ "en" ] +} + +## Play HTTP settings +# ~~~~~ +play.http { + ## Router + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # Define the Router object to use for this application. + # This router will be looked up first when the application is starting up, + # so make sure this is the entry point. + # Furthermore, it's assumed your route file is named properly. + # So for an application router like `my.application.Router`, + # you may need to define a router file `conf/my.application.routes`. + # Default to Routes in the root package (aka "apps" folder) (and conf/routes) + #router = my.application.Router + #TO allow more data in request body + errorHandler = modules.ErrorHandler + actionCreator = modules.OnRequestHandler + parser { + maxMemoryBuffer = 50MB + maxDiskBuffer = 50MB + } + secret.key="courseservicenadnfrfnsdmcvdfjbnfrsjfksdvcsdnfvdsfjkrvhsmdv" +} + +## Netty Provider +# ~~~~~ +play.server { + + provider = "play.core.server.NettyServerProvider" + + netty { + # The number of event loop threads. 0 means let Netty decide, which by default will select 2 times the number of + # available processors. + eventLoopThreads = 30 + + log.wire = true + + # The transport to use, either jdk or native. + # Native socket transport has higher performance and produces less garbage but are only available on linux + transport = "native" + + # If you run Play on Linux, you can use Netty's native socket transport + # for higher performance with less garbage. + #transport = "native" + maxChunkSize = 30000000 + option { + + # Set whether connections should use TCP keep alive + child.SO_KEEPALIVE = true + + # Set whether the TCP no delay flag is set + # child.tcpNoDelay = false + + # Set the size of the backlog of TCP connections. The default and exact meaning of this parameter is JDK specific. + # backlog = 100 + } + + http { + # The idle timeout for an open connection after which it will be closed + # Set to null or "infinite" to disable the timeout, but notice that this + # is not encouraged since timeout are important mechanisms to protect your + # servers from malicious attacks or programming mistakes. + idleTimeout = infinite + } + } +} + +## WS (HTTP Client) +# ~~~~~ +libraryDependencies += javaWs + +AuthenticationEnabled=true + +## Cache +# https://www.playframework.com/documentation/latest/JavaCache +# https://www.playframework.com/documentation/latest/ScalaCache +# ~~~~~ +# Play comes with an integrated cache API that can reduce the operational +# overhead of repeated requests. You must enable this by adding to build.sbt: +# +libraryDependencies += cache +# +play.cache { + # If you want to bind several caches, you can bind the individually + #bindCaches = ["db-cache", "user-cache", "session-cache"] +} + +# Logger +# ~~~~~ +# You can also configure logback (http://logback.qos.ch/), +# by providing an application-logger.xml file in the conf directory. + +# Root logger: +#logger.root=ERROR + +# Logger used by the framework: +#logger.play=INFO + +# Logger provided to your application: +#logger.application=DEBUG + +# APP Specific config +# ~~~~~ +# Application specific configurations can be provided here +play.modules { + enabled += modules.StartModule + enabled += modules.ActorStartModule + enabled += modules.StorageModule +} + +play.filters { + hosts { + # Allow requests to example.com, its subdomains, and localhost:9000. + allowed = ["localhost:9000","."] + } + enabled += filters.AccessLogFilter + enabled += filters.CustomGzipFilter + enabled += filters.ResponseFilter + disabled += play.filters.csrf.CSRFFilter +} +# Assessment Aggregator Configuration +assessment_direct_aggregation_enabled=true +kafka_assessment_topic="sunbird.assessment.raw" +kafka_topics_contentstate_invalid="sunbird.contentstate.invalid" +assessment_skip_missing_records=true + +# Redis optional support — master switch +# When false (default), Redis is not required. All Redis use cases are either skipped or fall back to in-memory alternatives. +redis.enabled = false +# Viewer module — Cassandra configuration +sunbird_course_keyspace = "sunbird_courses" +assessment_aggregator_table = "assessment_aggregator" +user_activity_agg_table = "user_activity_agg" + +# Feature flags +viewer_enabled = true +assessment_direct_aggregation_enabled = true +user_activity_agg_enabled = true + +# Viewer dispatcher (for aggregator pool) +pekko.actor.viewer-dispatcher { + type = Dispatcher + executor = "thread-pool-executor" + thread-pool-executor { + core-pool-size-min = 8 + core-pool-size-max = 64 + max-pool-size-max = 128 + queue-size = -1 + } + throughput = 1 +} diff --git a/modules/viewer/service/conf/logback-test.xml b/modules/viewer/service/conf/logback-test.xml new file mode 100644 index 00000000..adfa02c6 --- /dev/null +++ b/modules/viewer/service/conf/logback-test.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/modules/viewer/service/conf/logback.xml b/modules/viewer/service/conf/logback.xml new file mode 100644 index 00000000..9f047abf --- /dev/null +++ b/modules/viewer/service/conf/logback.xml @@ -0,0 +1,80 @@ + + + + + + + + + + %d %msg%n + + + + + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSX + Etc/UTC + + timestamp + msg + lname + tname + [ignore] + [ignore] + + + + + + + + + + timestamp + msg + lname + tname + [ignore] + [ignore] + exception + + + 30 + 2048 + sun\.reflect\..*\.invoke.* + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/viewer/service/conf/routes b/modules/viewer/service/conf/routes new file mode 100644 index 00000000..9a5ecd98 --- /dev/null +++ b/modules/viewer/service/conf/routes @@ -0,0 +1,25 @@ +# Routes — Viewer Service +# This file defines all application routes (Higher priority routes first) +OPTIONS /*all @controllers.viewer.ViewController.preflight(all) + +# Health Check +GET /health @controllers.viewer.ViewController.health(request: play.mvc.Http.Request) + +# Granular view lifecycle APIs +POST /v1/view/start @controllers.viewer.ViewController.viewStart(request: play.mvc.Http.Request) +POST /v1/view/update @controllers.viewer.ViewController.viewUpdate(request: play.mvc.Http.Request) +POST /v1/view/end @controllers.viewer.ViewController.viewEnd(request: play.mvc.Http.Request) + +# Assessment lifecycle (api.view.assess / api.assessment.read) +POST /v1/assessment/submit @controllers.viewer.ViewController.assessmentSubmit(request: play.mvc.Http.Request) +POST /v1/assessment/read @controllers.viewer.ViewController.assessmentRead(request: play.mvc.Http.Request) + +# Recompute roll-up (admin/ops) — mirrors legacy /v1/activity/agg, targets the aggregator +POST /v1/view/agg @controllers.viewer.ViewAggregateController.agg(request: play.mvc.Http.Request) + +# Read + summary APIs +POST /v1/view/read @controllers.viewer.ViewController.viewRead(request: play.mvc.Http.Request) +POST /v1/summary/read @controllers.viewer.ViewSummaryController.summaryRead(request: play.mvc.Http.Request) +GET /v1/summary/list/:userId @controllers.viewer.ViewSummaryController.summaryList(userId: String, request: play.mvc.Http.Request) +GET /v1/summary/download/:userId @controllers.viewer.ViewSummaryController.summaryDownload(userId: String, request: play.mvc.Http.Request) +DELETE /v1/summary/delete/:userId @controllers.viewer.ViewSummaryController.summaryDelete(userId: String, request: play.mvc.Http.Request) diff --git a/modules/viewer/service/pom.xml b/modules/viewer/service/pom.xml new file mode 100644 index 00000000..9094a7b3 --- /dev/null +++ b/modules/viewer/service/pom.xml @@ -0,0 +1,706 @@ + + + + + org.sunbird + viewer + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + viewer-service-impl + play2 + Viewer Service Module + Viewer Service - Play Framework Application + + + + scalaz-bintray + Scalaz Bintray - releases + https://dl.bintray.com/scalaz/releases/ + + false + + + + + + + typesafe-releases-plugins + https://repo.typesafe.com/typesafe/releases/ + + false + + + + + + + + + com.google.guava + guava + ${guava.version} + + + + + com.google.inject + guice + ${guice.version} + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + + + com.google.inject.extensions + guice-servlet + ${guice.version} + + + + + org.ow2.asm + asm + ${asm.version} + + + org.ow2.asm + asm-commons + ${asm.version} + + + org.ow2.asm + asm-tree + ${asm.version} + + + org.ow2.asm + asm-util + ${asm.version} + + + + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.major.version} + ${jackson.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + ${jackson.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + ${jackson.version} + + + com.fasterxml.jackson.module + jackson-module-parameter-names + ${jackson.version} + + + + + org.playframework + play_${scala.major.version} + ${play2.version} + + + com.fasterxml.jackson.core + jackson-core + + + joda-time + joda-time + + + com.google.guava + guava + + + org.lz4 + lz4-java + + + org.lz4 + lz4-java + + + at.yawk.lz4 + lz4-java + + + + + org.playframework + play-guice_${scala.major.version} + ${play2.version} + + + com.google.inject + guice + + + com.google.inject.extensions + guice-assistedinject + + + + + org.playframework + play-filters-helpers_${scala.major.version} + ${play2.version} + + + org.playframework + play-netty-server_${scala.major.version} + ${play2.version} + runtime + + + io.netty + netty-codec-http + + + + + org.playframework + play-pekko-http-server_${scala.major.version} + ${play2.version} + + + + + com.google.inject + guice + ${guice.version} + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + + + + + io.netty + netty-common + ${netty.version} + + + io.netty + netty-buffer + ${netty.version} + + + io.netty + netty-transport + ${netty.version} + + + io.netty + netty-codec + ${netty.version} + + + io.netty + netty-handler + ${netty.version} + + + io.netty + netty-resolver + ${netty.version} + + + io.netty + netty-transport-native-unix-common + ${netty.version} + + + io.netty + netty-transport-native-epoll + ${netty.version} + + + io.netty + netty-transport-classes-epoll + ${netty.version} + + + io.netty + netty-codec-dns + ${netty.version} + + + io.netty + netty-resolver-dns + ${netty.version} + + + io.netty + netty-codec-http + ${netty.version} + + + + + org.sunbird + viewer-actors + 1.0-SNAPSHOT + + + org.sunbird + course-actors-common + 1.0-SNAPSHOT + + + cassandra-unit + org.cassandraunit + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + org.ow2.asm + asm + + + org.ow2.asm + asm-commons + + + org.ow2.asm + asm-analysis + + + org.ow2.asm + asm-tree + + + org.ow2.asm + asm-util + + + com.google.inject.extensions + guice-multibindings + + + + + org.sunbird + course-actors + 1.0-SNAPSHOT + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + com.google.inject.extensions + guice-multibindings + + + + + org.sunbird + enrolment-actor + 1.0-SNAPSHOT + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + com.google.inject.extensions + guice-multibindings + + + + + org.sunbird + activity-aggregator + 1.0-SNAPSHOT + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + com.google.inject.extensions + guice-multibindings + + + + + org.sunbird + actor-util + 1.0-SNAPSHOT + + + org.sunbird + sunbird-actor-utils + 1.0-SNAPSHOT + + + org.sunbird + sunbird-platform-common + 1.0-SNAPSHOT + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + + + org.sunbird + assessment-aggregator + 1.0-SNAPSHOT + + + org.lz4 + lz4-java + + + at.yawk.lz4 + lz4-java + + + + + + + com.typesafe + config + 1.3.0 + + + org.scala-lang + scala-library + ${scala.version} + + + com.github.danielwegener + logback-kafka-appender + 0.2.0-RC2 + + + org.lz4 + lz4-java + + + at.yawk.lz4 + lz4-java + + + + + net.logstash.logback + logstash-logback-encoder + ${logstash-logback-encoder.version} + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + + + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-api + 2.17.1 + + + org.apache.logging.log4j + log4j-core + 2.17.1 + + + org.apache.logging.log4j + log4j-to-slf4j + 2.17.1 + + + + + org.ow2.asm + asm + ${asm.version} + + + org.ow2.asm + asm-commons + ${asm.version} + + + org.ow2.asm + asm-analysis + ${asm.version} + + + org.ow2.asm + asm-tree + ${asm.version} + + + org.ow2.asm + asm-util + ${asm.version} + + + + + org.playframework + play-specs2_${scala.major.version} + ${play2.version} + test + + + com.google.code.gson + gson + + + com.google.guava + guava + + + byte-buddy + net.bytebuddy + + + + + org.apache.pekko + pekko-testkit_${scala.major.version} + ${pekko.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + junit + junit + 4.13.1 + test + + + + + viewer-service-${project.version} + ${basedir}/app + ${basedir}/test + + + + ${basedir}/conf + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + + + com.google.code.play2-maven-plugin + play2-maven-plugin + ${play2.plugin.version} + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0 + + false + @{argLine} -Xmx1024m + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.text=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + --add-opens java.base/java.nio.charset=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/sun.net.www.protocol.jar=ALL-UNNAMED + + **/*Spec.java + **/*Test.java + + + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} + + ${project.build.directory}/jacoco.exec + ${project.build.directory}/jacoco.exec + + + + jacoco-initialize + + prepare-agent + + + + jacoco-site + package + + report + + + + + + + + + + + azure + + true + + + + org.sunbird + cloud-storage-sdk-azure + 2.0.1 + runtime + + + + + aws + + + org.sunbird + cloud-storage-sdk-aws + 2.0.1 + runtime + + + + + gcp + + + org.sunbird + cloud-storage-sdk-gcp + 2.0.1 + runtime + + + + + oci + + + org.sunbird + cloud-storage-sdk-oci + 2.0.1 + runtime + + + + + diff --git a/pom.xml b/pom.xml index bb7a4566..e7117252 100644 --- a/pom.xml +++ b/pom.xml @@ -98,12 +98,22 @@ + + + viewer + + modules/lms + modules/viewer + + + lern modules/userorg modules/lms + modules/viewer modules/notification modules/lern/service lern-jacoco-report diff --git a/scripts/build-local.sh b/scripts/build-local.sh index 36d5a1ca..850cc837 100755 --- a/scripts/build-local.sh +++ b/scripts/build-local.sh @@ -39,6 +39,7 @@ declare -A SERVICE_CONFIG=( [userorg]="userorg|modules/userorg/controller|userorg-service-1.0-SNAPSHOT-dist.zip" [lms]="lms|modules/lms/service|lms-service-1.0-SNAPSHOT-dist.zip" [notification]="notification|modules/notification/service|notification-service-1.0-SNAPSHOT-dist.zip" + [viewer]="viewer|modules/viewer/service|viewer-service-1.0-SNAPSHOT-dist.zip" ) # Help message @@ -46,7 +47,7 @@ function show_help { echo "Usage: $0 --service [options]" echo "" echo "Required:" - echo " -s, --service Service name: lern, userorg, lms, notification" + echo " -s, --service Service name: lern, userorg, lms, notification, viewer" echo "" echo "Options:" echo " -c, --csp Cloud Storage Provider (default: azure)" @@ -82,7 +83,7 @@ fi # Validate service name if [ -z "${SERVICE_CONFIG[$SERVICE]}" ]; then echo "Error: Unknown service '$SERVICE'" - echo "Valid services: lern, userorg, lms, notification" + echo "Valid services: lern, userorg, lms, notification, viewer" exit 1 fi diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 917e4e62..96090b86 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -17,7 +17,7 @@ # ./scripts/docker-build.sh --service notification --repo ghcr.io/myorg --push # # Options: -# -s, --service Service name (required): lern, userorg, lms, notification +# -s, --service Service name (required): lern, userorg, lms, notification, viewer # -r, --repo Docker registry/repository (optional, no push if omitted) # -n, --name Image name (default: service-service) # -t, --tag Image tag (default: latest) @@ -45,6 +45,7 @@ declare -A SERVICE_CONFIG=( [userorg]="userorg-service|build/userorg/Dockerfile|modules/userorg/controller/target/userorg-service-1.0-SNAPSHOT-dist.zip" [lms]="lms-service|build/lms/Dockerfile|modules/lms/service/target/lms-service-1.0-SNAPSHOT-dist.zip" [notification]="notification-service|build/notification/Dockerfile|modules/notification/service/target/notification-service-1.0-SNAPSHOT-dist.zip" + [viewer]="viewer-service|build/viewer/Dockerfile|modules/viewer/service/target/viewer-service-1.0-SNAPSHOT-dist.zip" ) # Help message @@ -52,7 +53,7 @@ function show_help { echo "Usage: $0 --service [options]" echo "" echo "Required:" - echo " -s, --service Service name: lern, userorg, lms, notification" + echo " -s, --service Service name: lern, userorg, lms, notification, viewer" echo "" echo "Options:" echo " -r, --repo Docker registry/repository (e.g., ghcr.io/myorg)" @@ -95,7 +96,7 @@ fi # Validate service name if [ -z "${SERVICE_CONFIG[$SERVICE]}" ]; then echo "Error: Unknown service '$SERVICE'" - echo "Valid services: lern, userorg, lms, notification" + echo "Valid services: lern, userorg, lms, notification, viewer" exit 1 fi