Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 2 additions & 13 deletions www/controllers/Gpg.php
Original file line number Diff line number Diff line change
Expand Up @@ -450,23 +450,12 @@ public function importFromUrl(string $url) : array
throw new Exception('Invalid URL');
}

// Check if the URL is reachable
try {
$this->httpRequestController->reachable([
'url' => $url,
'connectTimeout' => 5,
'proxy' => PROXY ?? null,
]);
} catch (Exception $e) {
throw new Exception('URL ' . $url . ' is not reachable: ' . $e->getMessage());
}

// Download GPG key
try {
$output = $this->httpRequestController->get([
'url' => $url,
'connectTimeout' => 5,
'timeout' => 10,
'connectTimeout' => 10,
'timeout' => 30,
'proxy' => PROXY ?? null,
'returnOutput' => true
]);
Expand Down
61 changes: 59 additions & 2 deletions www/controllers/Repo/Metadata/Rpm.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,65 @@ class Rpm extends Metadata
private $root;
private $createrepo = '/usr/bin/createrepo_c';
private $createrepoArgs = '-v --compress-type=gz --general-compress-type=gz';
// private $modifyrepo = '/usr/bin/modifyrepo_c';
private $modifyrepo = '/usr/bin/modifyrepo_c';
private $modifyrepoArgs = '--compress-type=gz';

public function setRoot(string $root)
{
$this->root = $root;
}

/**
* Add an additional metadata file to the repository metadata using modifyrepo_c
* Searches for the file with various compressions and deletes it afterwards
*/
private function addMetadata(string $filePrefix, string $mdtype): void
{
// Check if modifyrepo_c is available
if (!file_exists($this->modifyrepo)) {
throw new Exception('Could not find modifyrepo_c on the system');
}

// Look for the metadata file with various compression formats (and plain format) and use the first one found
$metadataFile = null;
foreach ([$filePrefix, $filePrefix . '.gz', $filePrefix . '.xz', $filePrefix . '.bz2', $filePrefix . '.zst'] as $file) {
if (file_exists($this->root . '/' . $file)) {
$metadataFile = $file;
break;
}
}

// If file doesn't exist, return silently (file may not exist in the source repo)
if (empty($metadataFile)) {
return;
}

// Add the file to the repository metadata using modifyrepo_c
$modifyrepoProcess = new Process($this->modifyrepo . ' ' . $this->modifyrepoArgs . ' --mdtype=' . $mdtype . ' ' . $this->root . '/' . $metadataFile . ' ' . $this->root . '/repodata/');
$modifyrepoProcess->setBackground(true);
$modifyrepoProcess->execute();

/**
* Retrieve PID of the launched process
* Then write PID to main PID file
*/
$this->taskController->addsubpid($modifyrepoProcess->getPid());

// Retrieve output from process
$this->taskLogSubStepController->output($modifyrepoProcess->getOutput(), 'pre');

if ($modifyrepoProcess->getExitCode() != 0) {
throw new Exception('Could not add ' . $mdtype . ' to repository metadata');
}

$modifyrepoProcess->close();

// Delete the file as it's now part of the metadata
if (!unlink($this->root . '/' . $metadataFile)) {
throw new Exception('Could not delete ' . $this->root . '/' . $metadataFile);
}
}

/**
* Create metadata files
*/
Expand Down Expand Up @@ -61,8 +113,13 @@ public function create(): void

$myprocess->close();

// Add updateinfo to metadata if it exists
$this->taskLogSubStepController->new('add-updateinfo', 'ADDING UPDATEINFO TO METADATA');
$this->addMetadata('updateinfo.xml', 'updateinfo');
$this->taskLogSubStepController->completed();

// Delete temporary metadata files as they are no longer needed
foreach (['comps.xml', 'updateinfo.xml', 'modules.yaml'] as $file) {
foreach (['comps.xml', 'modules.yaml'] as $file) {
if (file_exists($this->root . '/' . $file)) {
if (!unlink($this->root . '/' . $file)) {
throw new Exception('Could not delete ' . $this->root . '/' . $file);
Expand Down
8 changes: 4 additions & 4 deletions www/controllers/Repo/Mirror/Deb.php
Original file line number Diff line number Diff line change
Expand Up @@ -339,8 +339,8 @@ private function parsePackagesIndiceFile(string $url) : void
}

// Filter packages to keep only the X latest versions if 'keep-latest' is set
if (!empty($this->advancedParams['keep-latest'])) {
$this->debPackagesLocation = $this->keepLatestVersions($this->debPackagesLocation, (int) $this->advancedParams['keep-latest']);
if (!empty($this->advancedParams['packages']['keep-latest'])) {
$this->debPackagesLocation = $this->keepLatestVersions($this->debPackagesLocation, (int) $this->advancedParams['packages']['keep-latest']);
}

$this->taskLogSubStepController->completed(count($this->debPackagesLocation) . ' package' . (count($this->debPackagesLocation) > 1 ? 's' : '') . ' found');
Expand Down Expand Up @@ -502,8 +502,8 @@ private function parseSourcesIndiceFile(string $url) : void
}

// Filter source packages to keep only the X latest versions if 'keep-latest' is set
if (!empty($this->advancedParams['keep-latest'])) {
$this->sourcesPackagesLocation = $this->keepLatestVersions($this->sourcesPackagesLocation, (int) $this->advancedParams['keep-latest']);
if (!empty($this->advancedParams['packages']['keep-latest'])) {
$this->sourcesPackagesLocation = $this->keepLatestVersions($this->sourcesPackagesLocation, (int) $this->advancedParams['packages']['keep-latest']);
}

$this->taskLogSubStepController->completed(count($this->sourcesPackagesLocation) . ' source package' . (count($this->sourcesPackagesLocation) > 1 ? 's' : '') . ' found');
Expand Down
99 changes: 42 additions & 57 deletions www/controllers/Repo/Mirror/Rpm.php
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@ private function downloadComps(string $url) : void

$this->taskLogSubStepController->new('getting-comps', 'GETTING COMPS.XML', 'From ' . $url . '/' . $this->compsLocation);

// If sync of comps.xml is disabled, skip it
if ($this->advancedParams['metadata-sync']['comps'] == 'false') {
$this->taskLogSubStepController->completed('Skipping (sync disabled)');
return;
}

if (!$this->download($url . '/' . $this->compsLocation, $this->workingDir . '/comps.xml')) {
throw new Exception('Could not download <code>comps.xml</code>');
}
Expand All @@ -91,6 +97,12 @@ private function downloadModules(string $url) : void

$this->taskLogSubStepController->new('getting-modules', 'GETTING MODULES', 'From ' . $url . '/' . $this->modulesLocation);

// If sync of modules.yaml is disabled, skip it
if ($this->advancedParams['metadata-sync']['modules'] == 'false') {
$this->taskLogSubStepController->completed('Skipping (sync disabled)');
return;
}

// Get modules file extension
$modulesFileExtension = pathinfo($this->modulesLocation, PATHINFO_EXTENSION);

Expand All @@ -100,14 +112,10 @@ private function downloadModules(string $url) : void
}

/**
* We'll give this modules file a temporary name, to avoid it being included automatically by createrepo_c (it fails every time with modules.yaml file)
* It will be renamed and imported by modifyrepo_c later
* Download the modules file with a temporary compressed name, then uncompress it to a plain
* modules.yaml file so it can be found and included automatically by createrepo_c.
*/
$modulesFileExtension == 'gz' ? $modulesFileTargetName = 'modules.yaml.gz' : null;
$modulesFileExtension == 'xz' ? $modulesFileTargetName = 'modules.yaml.xz' : null;
$modulesFileExtension == 'bz2' ? $modulesFileTargetName = 'modules.yaml.bz2' : null;
$modulesFileExtension == 'zst' ? $modulesFileTargetName = 'modules.yaml.zst' : null;
$modulesFileExtension == 'yaml' ? $modulesFileTargetName = 'modules.yaml' : null;
$modulesFileTargetName = 'modules.yaml' . ($modulesFileExtension != 'yaml' ? '.' . $modulesFileExtension : '');

// Download modules file
if (!$this->download($url . '/' . $this->modulesLocation, $this->workingDir . '/' . $modulesFileTargetName)) {
Expand All @@ -119,29 +127,29 @@ private function downloadModules(string $url) : void
throw new Exception('<code>' . $modulesFileTargetName . '</code> checksum does not match provided checksum');
}

// If modules file has been downloaded as a .gz file, uncompress it (otherwise it will fail to be included to the metadata)
// Uncompress the modules file to a plain modules.yaml based on its actual mime type (required for createrepo_c to include it)
$mime = mime_content_type($this->workingDir . '/' . $modulesFileTargetName);

try {
if ($modulesFileExtension == 'gz') {
if ($mime == 'application/gzip') {
Gzip::uncompress($this->workingDir . '/' . $modulesFileTargetName);
}

if ($modulesFileExtension == 'bz2') {
} elseif ($mime == 'application/x-bzip2') {
Bzip2::uncompress($this->workingDir . '/' . $modulesFileTargetName, $this->workingDir . '/modules.yaml');
}

if ($modulesFileExtension == 'xz') {
} elseif ($mime == 'application/x-xz') {
Xz::uncompress($this->workingDir . '/' . $modulesFileTargetName, $this->workingDir . '/modules.yaml');
}

if ($modulesFileExtension == 'zst') {
} elseif ($mime == 'application/zstd') {
Zstd::uncompress($this->workingDir . '/' . $modulesFileTargetName);
} elseif ($mime == 'text/plain') {
// Already a plain yaml file, nothing to do
} else {
throw new Exception('MIME type not supported: ' . $mime . '. Please contact the developer to add support for this MIME type.');
}
} catch (Exception $e) {
throw new Exception('Error while uncompressing <code>' . $modulesFileTargetName . '</code><br><pre class="codeblock">' . $e->getMessage() . '</pre>');
}

// Delete original file (only if it was compressed and not already a plain .yaml file)
if ($modulesFileExtension != 'yaml') {
// Delete original compressed file if it was not already a plain yaml file
if ($mime != 'text/plain' and file_exists($this->workingDir . '/' . $modulesFileTargetName)) {
if (!unlink($this->workingDir . '/' . $modulesFileTargetName)) {
throw new Exception('Could not delete <code>' . $this->workingDir . '/' . $modulesFileTargetName . '</code>');
}
Expand All @@ -162,6 +170,12 @@ private function downloadUpdateInfo(string $url) : void

$this->taskLogSubStepController->new('getting-updateinfo', 'GETTING UPDATEINFO.XML.GZ', 'From ' . $url . '/' . $this->updateInfoLocation);

// If sync of updateinfo.xml.gz is disabled, skip it
if ($this->advancedParams['metadata-sync']['updateinfo'] == 'false') {
$this->taskLogSubStepController->completed('Skipping (sync disabled)');
return;
}

// Get updateinfo file extension
$updateInfoFileExtension = pathinfo($this->updateInfoLocation, PATHINFO_EXTENSION);

Expand All @@ -170,51 +184,22 @@ private function downloadUpdateInfo(string $url) : void
throw new Exception('Unsupported file extension <code>' . $updateInfoFileExtension . '</code> for <code>updateinfo</code> file. Please contact the developer to add support for this file extension.');
}

// We'll give this updateinfo file a target name according to its file extension
$updateInfoFileExtension == 'gz' ? $updateInfoFileTargetName = 'updateinfo.xml.gz' : null;
$updateInfoFileExtension == 'xz' ? $updateInfoFileTargetName = 'updateinfo.xml.xz' : null;
$updateInfoFileExtension == 'bz2' ? $updateInfoFileTargetName = 'updateinfo.xml.bz2' : null;
$updateInfoFileExtension == 'zst' ? $updateInfoFileTargetName = 'updateinfo.xml.zst' : null;
$updateInfoFileExtension == 'xml' ? $updateInfoFileTargetName = 'updateinfo.xml' : null;
/**
* Keep the file in its original (compressed) state, but give it a fixed target name so it can be
* found and added to the metadata by modifyrepo_c later. No need to uncompress it.
*/
$updateInfoFileTargetName = 'updateinfo.xml' . ($updateInfoFileExtension != 'xml' ? '.' . $updateInfoFileExtension : '');

// Download updateinfo file
if (!$this->download($url . '/' . $this->updateInfoLocation, $this->workingDir . '/' . $updateInfoFileTargetName)) {
throw new Exception('Could not download <code>updateinfo.xml.gz</code>');
throw new Exception('Could not download <code>' . $updateInfoFileTargetName . '</code>');
}

// Check that downloaded file checksum is the same as the provided checksum from repomd.xml
if (!$this->checksum($this->workingDir . '/' . $updateInfoFileTargetName, $this->updateInfoChecksum)) {
throw new Exception('<code>' . $updateInfoFileTargetName . '</code> checksum does not match provided checksum');
}

// If updateinfo file has been downloaded as a .gz file, uncompress it
try {
if ($updateInfoFileExtension == 'gz') {
Gzip::uncompress($this->workingDir . '/' . $updateInfoFileTargetName);
}

if ($updateInfoFileExtension == 'bz2') {
Bzip2::uncompress($this->workingDir . '/' . $updateInfoFileTargetName, $this->workingDir . '/updateinfo.xml');
}

if ($updateInfoFileExtension == 'xz') {
Xz::uncompress($this->workingDir . '/' . $updateInfoFileTargetName, $this->workingDir . '/updateinfo.xml');
}

if ($updateInfoFileExtension == 'zst') {
Zstd::uncompress($this->workingDir . '/' . $updateInfoFileTargetName);
}
} catch (Exception $e) {
throw new Exception('Error while uncompressing <code>' . $updateInfoFileTargetName . '</code><br><pre class="codeblock">' . $e->getMessage() . '</pre>');
}

// Delete original file (only if it was compressed and not already a plain .xml file)
if ($updateInfoFileExtension != 'xml') {
if (!unlink($this->workingDir . '/' . $updateInfoFileTargetName)) {
throw new Exception('Could not delete <code>' . $this->workingDir . '/' . $updateInfoFileTargetName . '</code>');
}
}

$this->taskLogSubStepController->completed();
}

Expand Down Expand Up @@ -473,8 +458,8 @@ private function parsePrimaryPackagesList(string $primaryFile) : void
}

// Filter packages to keep only the X latest versions if 'keep-latest' is set
if (!empty($this->advancedParams['keep-latest'])) {
$this->rpmPackagesLocation = $this->keepLatestVersions($this->rpmPackagesLocation, (int) $this->advancedParams['keep-latest']);
if (!empty($this->advancedParams['packages']['keep-latest'])) {
$this->rpmPackagesLocation = $this->keepLatestVersions($this->rpmPackagesLocation, (int) $this->advancedParams['packages']['keep-latest']);
}

$this->taskLogSubStepController->completed(count($this->rpmPackagesLocation) . ' package' . (count($this->rpmPackagesLocation) > 1 ? 's' : '') . ' found');
Expand Down
5 changes: 5 additions & 0 deletions www/controllers/Task/Form/Create.php
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ public function validate(array $formParams): void
if ($formParams['package-type'] == 'rpm') {
// Check releasever
Param\Releasever::check($formParams['releasever']);

// Check additional metadata files sync
Param\Metadata::checkSync($formParams['advanced-params']['metadata-sync']['comps']);
Param\Metadata::checkSync($formParams['advanced-params']['metadata-sync']['modules']);
Param\Metadata::checkSync($formParams['advanced-params']['metadata-sync']['updateinfo']);
}

// Case package type is 'deb'
Expand Down
7 changes: 7 additions & 0 deletions www/controllers/Task/Form/Param/Metadata.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@

class Metadata
{
public static function checkSync(string $sync): void
{
if (!in_array($sync, ['true', 'false'])) {
throw new Exception('Invalid value for metadata sync parameter. Must be "true" or "false".');
}
}

public static function checkOrigin(string $origin): void
{
if (empty($origin)) {
Expand Down
7 changes: 7 additions & 0 deletions www/controllers/Task/Form/Update.php
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ public function validate(array $formParams): void
// Check gpg check
Param\GpgCheck::check($formParams['gpg-check']);

if ($repoController->getPackageType() == 'rpm') {
// Check additional metadata files sync
Param\Metadata::checkSync($formParams['advanced-params']['metadata-sync']['comps']);
Param\Metadata::checkSync($formParams['advanced-params']['metadata-sync']['modules']);
Param\Metadata::checkSync($formParams['advanced-params']['metadata-sync']['updateinfo']);
}

if ($repoController->getPackageType() == 'deb') {
// Check metadata custom fields
Param\Metadata::checkOrigin($formParams['advanced-params']['metadata-custom-fields']['origin']);
Expand Down
1 change: 1 addition & 0 deletions www/public/resources/styles/common.css
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ pre.codeblock {
.justify-items-center{justify-items: center}
.justify-items-end{justify-items: flex-end}
.justify-self-center{justify-self:center}
.justify-self-end{justify-self:flex-end}
.align-content-center {align-content: center}
.align-content-right {align-content: flex-end}
.align-item-start {align-items: start !important}
Expand Down
2 changes: 1 addition & 1 deletion www/version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
5.13.1
5.13.2
2 changes: 1 addition & 1 deletion www/views/includes/containers/tasks/log/substep.inc.php
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
} else if ($type == 'error') {
echo '<p class="redtext">' . $message . '</p>';
} else if ($type == 'pre') {
echo '<pre class="codeblock">' . $message . '</pre>';
echo '<pre class="codeblock copy">' . $message . '</pre>';
} else {
echo '<p>' . $message . '</p>';
}
Expand Down
Loading
Loading