Skip to content

Batch moodle:sync Moodle writes across users to stop stacking #113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Celeo merged 1 commit into from
Jul 21, 2026
Merged

Batch moodle:sync Moodle writes across users to stop stacking #113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions app/Classes/VATUSAMoodle.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ class VATUSAMoodle extends MoodleRest
/** @var array|null Memoized result of getCategories() for the lifetime of this instance */
private $categoriesCache = null;

/** @var array Memoized results of getCoursesInCategory(), keyed by category id */
private $coursesInCategoryCache = [];

/** @var int Seconds to bound each Moodle HTTP request to */
private const HTTP_TIMEOUT_SECONDS = 30;

Expand Down Expand Up @@ -400,6 +403,26 @@ public function updateUser(User $user, int $id)
], self::METHOD_POST);
}

/**
* Bulk update Users
*
* @param array $items Each item: ['id' => int, 'fname' => string, 'lname' => string, 'email' => string]
*
* @return mixed
* @throws Exception
*/
public function updateUsersBulk(array $items)
{
return $this->request("core_user_update_users", [
'users' => array_values(array_map(fn ($item) => [
'id' => $item['id'],
'firstname' => $item['fname'],
'lastname' => $item['lname'],
'email' => $item['email']
], $items))
], self::METHOD_POST);
}

/**
* Create Cohort
*
Expand Down Expand Up @@ -673,9 +696,14 @@ function getContext(
function getCoursesInCategory(
int $catid = null
) {
$params = $catid ? ["field" => "category", "value" => $catid] : [];
$key = $catid ?? 0;
if (!array_key_exists($key, $this->coursesInCategoryCache)) {
$params = $catid ? ["field" => "category", "value" => $catid] : [];
$this->coursesInCategoryCache[$key] = $this->request("core_course_get_courses_by_field",
$params)["courses"];
}

return $this->request("core_course_get_courses_by_field", $params)["courses"];
return $this->coursesInCategoryCache[$key];
}

public
Expand Down
153 changes: 120 additions & 33 deletions app/Console/Commands/MoodleSync.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class MoodleSync extends Command
*
* @var string
*/
protected $signature = 'moodle:sync
protected $signature = 'moodle:sync
{user? : CID of a single user to sync}';

/**
Expand Down Expand Up @@ -56,47 +56,138 @@ public function handle()
return 0;
}

$this->sync($user);
$items = $this->computeSyncItems($user, $this->resolveMoodleId($user));
$this->flushItems($items);

return 0;
}

//Syncronize Users
$moodleIds = $this->moodle->getAllUserIdMap();
User::with('visits', 'academyCompetencies.course', 'roles')->chunk(1000, function ($users) use ($moodleIds) {
foreach ($users as $user) {
if ($moodleIds->has($user->cid)) {
$this->sync($user, (int) $moodleIds[$user->cid]);
logger()->info("moodle:sync starting bulk pass: {$moodleIds->count()} Moodle-linked users");

$chunkNum = 0;
$totals = ['users' => 0, 'updates' => 0, 'cohorts' => 0, 'roles' => 0, 'enrolments' => 0];

User::with('visits', 'academyCompetencies.course', 'roles')->chunk(1000,
function ($users) use ($moodleIds, &$chunkNum, &$totals) {
$chunkNum++;
$updates = $cohorts = $roles = $enrolments = [];

foreach ($users as $user) {
if (!$moodleIds->has($user->cid)) {
continue;
}
$items = $this->computeSyncItems($user, (int) $moodleIds[$user->cid]);
$updates[] = $items['update'];
$cohorts = array_merge($cohorts, $items['cohorts']);
$roles = array_merge($roles, $items['roles']);
$enrolments = array_merge($enrolments, $items['enrolments']);
}
}
});

$this->flushBulk($chunkNum, 'update', $updates, fn ($items) => $this->moodle->updateUsersBulk($items));
$this->flushBulk($chunkNum, 'cohorts', $cohorts,
fn ($items) => $this->moodle->assignCohortsBulk($items));
$this->flushBulk($chunkNum, 'roles', $roles, fn ($items) => $this->moodle->assignRolesBulk($items));
$this->flushBulk($chunkNum, 'enrolments', $enrolments,
fn ($items) => $this->moodle->enrolUsersBulk($items));

$totals['users'] += count($users);
$totals['updates'] += count($updates);
$totals['cohorts'] += count($cohorts);
$totals['roles'] += count($roles);
$totals['enrolments'] += count($enrolments);

logger()->info("moodle:sync chunk {$chunkNum}: " . count($users) . " users, "
. count($updates) . " updates, " . count($cohorts) . " cohorts, "
. count($roles) . " roles, " . count($enrolments) . " enrolments");
});

logger()->info("moodle:sync finished bulk pass: {$totals['users']} users seen, "
. "{$totals['updates']} updates, {$totals['cohorts']} cohorts, "
. "{$totals['roles']} roles, {$totals['enrolments']} enrolments across {$chunkNum} chunks");

return 0;
}

/**
* Synchronize Roles
* Resolve a user's Moodle id for the single-user CLI path (create-or-update decision).
*
* @param \App\User $user
* @param int|null $knownId Moodle user id, if already known (bulk sync path) — skips the
* redundant existence-check lookup that the single-user CLI path
* still needs to decide create-vs-update.
*
* @return int
* @throws \Exception
*/
private function resolveMoodleId(User $user): int
{
if ($id = $this->moodle->getUserId($user->cid)) {
return $id;
}

return $this->moodle->createUser($user)[0]["id"];
}

/**
* Flush one bulk call for a chunk, logging and rethrowing on failure so a mid-run
* failure points straight at which chunk and which call broke.
*
* @param int $chunkNum
* @param string $kind
* @param array $items
* @param callable $call
*
* @throws \Exception
*/
private function sync(User $user, ?int $knownId = null)
private function flushBulk(int $chunkNum, string $kind, array $items, callable $call)
{
//Update or Create
if ($knownId !== null) {
$id = $knownId;
$this->moodle->updateUser($user, $id);
} elseif ($id = $this->moodle->getUserId($user->cid)) {
//Update Information
$this->moodle->updateUser($user, $id);
} else {
//Create User
$id = $this->moodle->createUser($user)[0]["id"];
if (empty($items)) {
return;
}

try {
$call($items);
} catch (\Exception $e) {
logger()->error("moodle:sync chunk {$chunkNum}: {$kind} bulk call failed: {$e->getMessage()}");
throw $e;
}
}

/**
* Flush a single user's computed items immediately (single-user CLI path).
*
* @param array $items
*
* @throws \Exception
*/
private function flushItems(array $items)
{
$this->moodle->updateUsersBulk([$items['update']]);
if (!empty($items['cohorts'])) {
$this->moodle->assignCohortsBulk($items['cohorts']);
}
if (!empty($items['roles'])) {
$this->moodle->assignRolesBulk($items['roles']);
}
if (!empty($items['enrolments'])) {
$this->moodle->enrolUsersBulk($items['enrolments']);
}
}

/**
* Compute what needs to happen in Moodle for a user — cohorts, roles, and enrolments —
* without making any HTTP calls itself. Callers accumulate these across many users and
* flush them as bulk calls (whole-table sync), or flush a single user's items
* immediately (single-user CLI path). Cohort/role clearing stays per-user since it's a
* direct DB delete (cheap, not the HTTP bottleneck this split targets).
*
* @param \App\User $user
* @param int $id Moodle user id
*
* @return array{update: array, cohorts: array, roles: array, enrolments: array}
* @throws \Exception
*/
private function computeSyncItems(User $user, int $id): array
{
$facilities = $user->visits->pluck('facility')->merge(collect($user->facility))->unique();

//Assign Cohorts
Expand All @@ -119,10 +210,6 @@ private function sync(User $user, ?int $knownId = null)
$cohorts[] = $facility . "-V"; //Facility level visitor
$cohorts[] = "$facility-" . Helper::ratingShortFromInt($user->rating); //Facility level rating
}
if (!empty($cohorts)) {
$this->moodle->assignCohortsBulk(array_map(fn ($cnumber) => ['uid' => $id, 'cnumber' => $cnumber],
$cohorts));
}

//Clear Roles
$this->moodle->clearUserRoles($id);
Expand Down Expand Up @@ -203,11 +290,11 @@ private function sync(User $user, ?int $knownId = null)
$roles[] = ['uid' => $id, 'cid' => $this->moodle->getCategoryFromShort($user->facility, true), 'role' => "MTR", 'context' => "coursecat"];
}*/

if (!empty($roles)) {
$this->moodle->assignRolesBulk($roles);
}
if (!empty($enrolments)) {
$this->moodle->enrolUsersBulk($enrolments);
}
return [
'update' => ['id' => $id, 'fname' => $user->fname, 'lname' => $user->lname, 'email' => $user->email],
'cohorts' => array_map(fn ($cnumber) => ['uid' => $id, 'cnumber' => $cnumber], $cohorts),
'roles' => $roles,
'enrolments' => $enrolments,
];
}
}
Loading