diff --git a/classes/db/stepdata_migration_utils.php b/classes/db/stepdata_migration_utils.php index cbc86d9..3e7868a 100644 --- a/classes/db/stepdata_migration_utils.php +++ b/classes/db/stepdata_migration_utils.php @@ -70,14 +70,16 @@ public static function to_new_name( return qtype_matrix_question::responsekey($newrowindex, $newcolindex); } - public static function stepdata_sql(string $qinsql, string $stepdatanamelike): string { + public static function stepdata_sql(int $minqid, int $maxqid, string $stepdatanamelike): string { + $operator = str_contains($stepdatanamelike, '%') ? 'LIKE' : '='; return "SELECT qasd.id as stepdataid, q.id as questionid, qa.id as attemptid, qas.id as stepid, qasd.name, qasd.value FROM {question} q JOIN {question_attempts} qa ON qa.questionid = q.id JOIN {question_attempt_steps} qas ON qas.questionattemptid = qa.id JOIN {question_attempt_step_data} qasd ON qasd.attemptstepid = qas.id - WHERE qasd.name LIKE '" . $stepdatanamelike . "' - AND q.id " . $qinsql . " + WHERE qasd.name ".$operator." '" . $stepdatanamelike . "' + AND q.qtype = 'matrix' + AND q.id BETWEEN ".$minqid." AND ".$maxqid." ORDER BY qasd.id "; } @@ -87,7 +89,7 @@ public static function extract_matrixinfos(array $questionids):array { $matrixinfos = []; - [$qinsql, $qidparams] = $DB->get_in_or_equal(array_keys($questionids)); + [$qinsql, $qidparams] = $DB->get_in_or_equal($questionids); // Leave out matrix questions with broken data (missing col records) $colssql = " SELECT qmc.id as colid, qm.id as matrixid, q.id as questionid @@ -119,9 +121,10 @@ public static function extract_matrixinfos(array $questionids):array { $matrixcols->close(); // Collect info about attempt row order. - $orderstepdatasql = stepdata_migration_utils::stepdata_sql($qinsql, '_order'); - // FIXME: Should probably be done in batches of 100.000 - $orderdatars = $DB->get_recordset_sql($orderstepdatasql, $qidparams); + $minqid = min($questionids); + $maxqid = max($questionids); + $orderstepdatasql = stepdata_migration_utils::stepdata_sql($minqid, $maxqid, '_order'); + $orderdatars = $DB->get_recordset_sql($orderstepdatasql); foreach ($orderdatars as $orderdata) { if (!$orderdata->questionid || !isset($matrixinfos[$orderdata->questionid])) { continue; @@ -137,12 +140,14 @@ public static function extract_matrixinfos(array $questionids):array { public static function migrate_stepdata(array &$matrixinfos, array $questionids):void { global $DB; - [$qinsql, $qidparams] = $DB->get_in_or_equal(array_keys($questionids)); - $cellstepdatasql = self::stepdata_sql($qinsql, 'cell%'); - // FIXME: Should probably be done in batches of 100.000 - - $celldataupdatechunksize = 10000; - $celldatars = $DB->get_recordset_sql($cellstepdatasql, $qidparams); + $minqid = min($questionids); + $maxqid = max($questionids); + $cellstepdatasql = self::stepdata_sql($minqid, $maxqid, 'cell%'); + + // Lower update chunks lead to lower times for the update query. + // This seems to be caused by using lots of CASE statements. + $celldataupdatechunksize = 100; + $celldatars = $DB->get_recordset_sql($cellstepdatasql); $nrprocessedchunkcelldata = 0; $celldataids = []; $sqlnamecase = ''; @@ -190,5 +195,6 @@ public static function migrate_stepdata(array &$matrixinfos, array $questionids) } } $celldatars->close(); + unset($celldatars); } } diff --git a/db/upgrade.php b/db/upgrade.php index d575d19..e4f6282 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -162,22 +162,29 @@ function xmldb_qtype_matrix_upgrade(int $oldversion): bool { if ($total) { core_php_time_limit::raise(); // Ensure we have a base memory limit with which to work. + // This should at least amount to 2GB RAM for this process. raise_memory_limit(MEMORY_HUGE); $now = time(); - // Guessed batch size for processing questions when updating question attempt step data. - $questionbatchsize = 1000; + // Batch size for processing questions when updating question attempt step data. + // The given batch size led to 1.4 GB of the process being matrixinfos. + // This was deemed okay because the recordsets do not take much memory. + $questionbatchsize = 25000; // Show a progress bar. $pbar = new progress_bar('upgrade_qtype_matrix_stepdata_to_row', 500, true); $offset = 0; while ($offset < $total) { $pbar->update($offset, $total, "Updating attempt data for qtype_matrix questions - $offset/$total questions."); - $questionids = $DB->get_records( + $questionids = array_keys($DB->get_records( 'question', ['qtype' => 'matrix'], 'id ASC', 'id', $offset, $questionbatchsize - ); + )); $offset += $questionbatchsize; $matrixinfos = stepdata_migration_utils::extract_matrixinfos($questionids); // Now migrate the cell stepdata for the question batch (also done in batches). stepdata_migration_utils::migrate_stepdata($matrixinfos, $questionids); + // Necessary to prevent out of memory errors because PHP didn't garbage collect early enough. + unset($matrixinfos); + gc_collect_cycles(); + gc_mem_caches(); } $pbar->update($offset, $total, 'Done. Seconds: '.(time() - $now)); }