From 77683f233fc721d1b4fcfbb7a23e7510fddea156 Mon Sep 17 00:00:00 2001 From: SyZn <1295141+SyZn@users.noreply.github.com> Date: Sat, 24 Apr 2021 12:08:25 +0200 Subject: [PATCH 01/12] Update composer.json Added needed dependency --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 28fdd5c..4e37153 100644 --- a/composer.json +++ b/composer.json @@ -22,7 +22,8 @@ "heimrichhannot/contao-multi-column-editor-bundle": "^2.4", "heimrichhannot/contao-utils-bundle": "^2.135", "heimrichhannot/contao-list_widget": "^2.1", - "symfony/stopwatch": "^3.4 || ^4.4" + "symfony/stopwatch": "^3.4 || ^4.4", + "php-http/guzzle6-adapter": "^1.1" }, "require-dev": { "contao/test-case": "1.1.*", @@ -31,7 +32,6 @@ "phpunit/phpunit": ">=6.0 <6.5", "phpunit/phpunit-mock-objects": "^4.0|^5.0", "phpunit/php-token-stream": "^1.4|^2.0|^3.0", - "php-http/guzzle6-adapter": "^1.1", "php-http/message-factory": "^1.0.2", "satooshi/php-coveralls": "^2.0", "symfony/phpunit-bridge": "^3.2" From e39182bb1955d1f912202ca6e06be625550baea2 Mon Sep 17 00:00:00 2001 From: SyZn <1295141+SyZn@users.noreply.github.com> Date: Sat, 24 Apr 2021 15:14:50 +0200 Subject: [PATCH 02/12] Added new parameter for ignoring self-signed SSL-certs --- src/Source/AbstractFileSource.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Source/AbstractFileSource.php b/src/Source/AbstractFileSource.php index 28f681a..5e28b74 100644 --- a/src/Source/AbstractFileSource.php +++ b/src/Source/AbstractFileSource.php @@ -91,14 +91,14 @@ public function getFileContent(bool $cache = false): string $content = $this->getValueFromRemoteCache($cacheKey); if (empty($content)) { - $this->storeValueToRemoteCache($this->sourceModel->sourceUrl, $cacheKey, $this->sourceModel->httpMethod, $event->getAuth()); + $this->storeValueToRemoteCache($this->sourceModel->sourceUrl, $cacheKey, $this->sourceModel->httpMethod, $event->getAuth(), ["verify" => (!$this->sourceModel->dontCheckSSL)]); $content = $this->getValueFromRemoteCache($cacheKey); } break; } - $result = $this->getContentFromUrl($this->sourceModel->httpMethod, $this->sourceModel->sourceUrl, $event->getAuth()); + $result = $this->getContentFromUrl($this->sourceModel->httpMethod, $this->sourceModel->sourceUrl, $event->getAuth(), ["verify" => (!$this->sourceModel->dontCheckSSL))]); $content = $result['result']; break; From 208b7f12122ae6a670af2c0e18fbd2c6d51385bd Mon Sep 17 00:00:00 2001 From: SyZn <1295141+SyZn@users.noreply.github.com> Date: Sat, 24 Apr 2021 15:20:13 +0200 Subject: [PATCH 03/12] Added HTTPOptions parameter, fix ErrorHanding Added the parameter HTTPOptions to configure the GuzzleClient. Fixed the error handling to report better errors, when there is no response to the request. --- src/Source/AbstractSource.php | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Source/AbstractSource.php b/src/Source/AbstractSource.php index db801a6..71db276 100644 --- a/src/Source/AbstractSource.php +++ b/src/Source/AbstractSource.php @@ -100,17 +100,20 @@ protected function getMappedItemData(?array $element, array $mapping): array return $result; } - protected function getContentFromUrl(string $method, string $url, array $auth = []): array + protected function getContentFromUrl(string $method, string $url, array $auth = [], array $HTTPClientOptions = []): array { - $client = new Client(); + $client = new Client($HTTPClientOptions); try { $response = $client->request($method, \Contao\StringUtil::decodeEntities($url), $auth); } catch (RequestException $e) { - return [ - 'statusCode' => $e->getResponse()->getStatusCode(), - 'result' => $e->getResponse()->getBody()->getContents(), - ]; + if($e->hasResponse()){ + return [ + 'statusCode' => $e->getResponse()->getStatusCode(), + 'result' => $e->getResponse()->getBody()->getContents(), + ]; + } + throw $e; } return [ @@ -133,11 +136,11 @@ protected function deleteValueFromRemoteCache(string $cacheKey): string return $filesystemCache->deleteItem('entity-import-remote.'.$cacheKey); } - protected function storeValueToRemoteCache(string $url, string $cacheKey, string $method, array $auth = []) + protected function storeValueToRemoteCache(string $url, string $cacheKey, string $method, array $auth = [], array $HTTPClientOptions = []) { $filesystemCache = $this->getFilesystemCache(); - $response = $this->getContentFromUrl($method, $url, $auth); + $response = $this->getContentFromUrl($method, $url, $auth, $HTTPClientOptions); if (200 === $response['statusCode']) { $filesystemCache->set('entity-import-remote.'.$cacheKey, $response['result']); From 02e1e3f31a221bde727c76cc79a52fd6da0c9032 Mon Sep 17 00:00:00 2001 From: SyZn <1295141+SyZn@users.noreply.github.com> Date: Sat, 24 Apr 2021 15:24:39 +0200 Subject: [PATCH 04/12] Added don't check SSL field --- src/Resources/contao/dca/tl_entity_import_source.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Resources/contao/dca/tl_entity_import_source.php b/src/Resources/contao/dca/tl_entity_import_source.php index 15669f6..6c7f0ec 100644 --- a/src/Resources/contao/dca/tl_entity_import_source.php +++ b/src/Resources/contao/dca/tl_entity_import_source.php @@ -81,7 +81,7 @@ // Subpalettes 'subpalettes' => [ - 'retrievalType_http' => 'sourceUrl,httpMethod,httpAuth,fileType', + 'retrievalType_http' => 'sourceUrl,dontCheckSSL,httpMethod,httpAuth,fileType', 'retrievalType_contao_file_system' => 'fileSRC,fileType', 'retrievalType_absolute_path' => 'absolutePath', 'fileType_csv' => 'fileContent,csvHeaderRow,csvSkipEmptyLines,csvDelimiter,csvEnclosure,csvEscape,fieldMappingCopier,fieldMappingPresets,fieldMapping', @@ -253,6 +253,13 @@ 'eval' => ['submitOnChange' => true, 'includeBlankOption' => true, 'tl_class' => 'clr w50', 'rgxp' => 'url'], 'sql' => "varchar(255) NOT NULL default ''", ], + 'dontCheckSSL' => [ + 'label' => &$GLOBALS['TL_LANG']['tl_entity_import_source']['dontCheckSSL'], + 'exclude' => true, + 'inputType' => 'checkbox', + 'eval' => ['submitOnChange' => true], + 'sql' => "char(1) NOT NULL default ''", + ], 'httpMethod' => [ 'label' => &$GLOBALS['TL_LANG']['tl_entity_import_source']['httpMethod'], 'exclude' => true, From bc65d07f8c3ce9d9a72114cbc4ed06b2bdc27cd5 Mon Sep 17 00:00:00 2001 From: SyZn <1295141+SyZn@users.noreply.github.com> Date: Sat, 24 Apr 2021 15:41:55 +0200 Subject: [PATCH 05/12] Fix positioning --- src/Resources/contao/dca/tl_entity_import_source.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Resources/contao/dca/tl_entity_import_source.php b/src/Resources/contao/dca/tl_entity_import_source.php index 6c7f0ec..8593340 100644 --- a/src/Resources/contao/dca/tl_entity_import_source.php +++ b/src/Resources/contao/dca/tl_entity_import_source.php @@ -81,7 +81,7 @@ // Subpalettes 'subpalettes' => [ - 'retrievalType_http' => 'sourceUrl,dontCheckSSL,httpMethod,httpAuth,fileType', + 'retrievalType_http' => 'dontCheckSSL,sourceUrl,httpMethod,httpAuth,fileType', 'retrievalType_contao_file_system' => 'fileSRC,fileType', 'retrievalType_absolute_path' => 'absolutePath', 'fileType_csv' => 'fileContent,csvHeaderRow,csvSkipEmptyLines,csvDelimiter,csvEnclosure,csvEscape,fieldMappingCopier,fieldMappingPresets,fieldMapping', @@ -257,7 +257,7 @@ 'label' => &$GLOBALS['TL_LANG']['tl_entity_import_source']['dontCheckSSL'], 'exclude' => true, 'inputType' => 'checkbox', - 'eval' => ['submitOnChange' => true], + 'eval' => ['submitOnChange' => true, 'tl_class' => 'w50 m12'], 'sql' => "char(1) NOT NULL default ''", ], 'httpMethod' => [ From 6e53a9851e34dc297a38d7f87011405ffc328500 Mon Sep 17 00:00:00 2001 From: SyZn <1295141+SyZn@users.noreply.github.com> Date: Sat, 24 Apr 2021 15:43:01 +0200 Subject: [PATCH 06/12] Added translation for SSL-Field --- src/Resources/contao/languages/de/tl_entity_import_source.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Resources/contao/languages/de/tl_entity_import_source.php b/src/Resources/contao/languages/de/tl_entity_import_source.php index 88910b1..40ae0ad 100644 --- a/src/Resources/contao/languages/de/tl_entity_import_source.php +++ b/src/Resources/contao/languages/de/tl_entity_import_source.php @@ -55,6 +55,8 @@ $lang['retrievalType'][\HeimrichHannot\EntityImportBundle\DataContainer\EntityImportSourceContainer::RETRIEVAL_TYPE_CONTAO_FILE_SYSTEM] = 'Contao Dateiverwaltung'; $lang['retrievalType'][\HeimrichHannot\EntityImportBundle\DataContainer\EntityImportSourceContainer::RETRIEVAL_TYPE_ABSOLUTE_PATH] = 'Absoluter Pfad'; +$lang['dontCheckSSL'][0] = 'SSL nicht verifizieren'; +$lang['dontCheckSSL'][1] = 'Klicken sie diese Checkbox an, damit bei der Verbindung das SSL-Zertifikat nicht überprüft wird. (Nutzen sie diese Option nur, wenn sie wissen was sie tun!)'; $lang['sourceUrl'][0] = 'URL'; $lang['sourceUrl'][1] = 'Geben Sie hier die URL zur Datei ein.'; $lang['absolutePath'][0] = 'Absoluter Dateipfad'; From d9b91c21fe9adca13fca690f7fbbf37913a573cb Mon Sep 17 00:00:00 2001 From: SyZn <1295141+SyZn@users.noreply.github.com> Date: Sat, 24 Apr 2021 15:45:21 +0200 Subject: [PATCH 07/12] Added en translation of SSL-Field --- src/Resources/contao/languages/en/tl_entity_import_source.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Resources/contao/languages/en/tl_entity_import_source.php b/src/Resources/contao/languages/en/tl_entity_import_source.php index d06fa20..d9c8473 100644 --- a/src/Resources/contao/languages/en/tl_entity_import_source.php +++ b/src/Resources/contao/languages/en/tl_entity_import_source.php @@ -48,6 +48,8 @@ $lang['type'][\HeimrichHannot\EntityImportBundle\DataContainer\EntityImportSourceContainer::TYPE_DATABASE] = 'Database'; $lang['type'][\HeimrichHannot\EntityImportBundle\DataContainer\EntityImportSourceContainer::TYPE_FILE] = 'File'; +$lang['dontCheckSSL'][0] = 'Don\'t check SSL'; +$lang['dontCheckSSL'][1] = 'Activate this checkbox if you want to prevent checking of SSL-Certificates for this request. (Do this only if you know what this means!)'; $lang['sourceUrl'][0] = 'Url'; $lang['sourceUrl'][1] = 'Enter the URL to the file here.'; $lang['absolutePath'][0] = 'Absolute file path'; From c45e03030deebe22bf108962493b3b1fa326ebc2 Mon Sep 17 00:00:00 2001 From: Defcon0 Date: Mon, 26 Apr 2021 16:08:52 +0200 Subject: [PATCH 08/12] Update tl_entity_import_source.php --- src/Resources/contao/languages/de/tl_entity_import_source.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Resources/contao/languages/de/tl_entity_import_source.php b/src/Resources/contao/languages/de/tl_entity_import_source.php index 40ae0ad..f6f13c2 100644 --- a/src/Resources/contao/languages/de/tl_entity_import_source.php +++ b/src/Resources/contao/languages/de/tl_entity_import_source.php @@ -55,8 +55,8 @@ $lang['retrievalType'][\HeimrichHannot\EntityImportBundle\DataContainer\EntityImportSourceContainer::RETRIEVAL_TYPE_CONTAO_FILE_SYSTEM] = 'Contao Dateiverwaltung'; $lang['retrievalType'][\HeimrichHannot\EntityImportBundle\DataContainer\EntityImportSourceContainer::RETRIEVAL_TYPE_ABSOLUTE_PATH] = 'Absoluter Pfad'; -$lang['dontCheckSSL'][0] = 'SSL nicht verifizieren'; -$lang['dontCheckSSL'][1] = 'Klicken sie diese Checkbox an, damit bei der Verbindung das SSL-Zertifikat nicht überprüft wird. (Nutzen sie diese Option nur, wenn sie wissen was sie tun!)'; +$lang['dontCheckSSL'][0] = 'SSL-Verifikation deaktivieren'; +$lang['dontCheckSSL'][1] = 'Wählen Sie diese Option, damit bei der Verbindung das SSL-Zertifikat nicht überprüft wird. Nutzen sie diese Option nur, wenn Sie wissen was Sie tun!'; $lang['sourceUrl'][0] = 'URL'; $lang['sourceUrl'][1] = 'Geben Sie hier die URL zur Datei ein.'; $lang['absolutePath'][0] = 'Absoluter Dateipfad'; From c7ffc402853beae9124d595372d976c7f02211e9 Mon Sep 17 00:00:00 2001 From: Defcon0 Date: Mon, 26 Apr 2021 16:10:08 +0200 Subject: [PATCH 09/12] Update tl_entity_import_source.php --- src/Resources/contao/languages/en/tl_entity_import_source.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Resources/contao/languages/en/tl_entity_import_source.php b/src/Resources/contao/languages/en/tl_entity_import_source.php index d9c8473..5e35502 100644 --- a/src/Resources/contao/languages/en/tl_entity_import_source.php +++ b/src/Resources/contao/languages/en/tl_entity_import_source.php @@ -49,7 +49,7 @@ $lang['type'][\HeimrichHannot\EntityImportBundle\DataContainer\EntityImportSourceContainer::TYPE_FILE] = 'File'; $lang['dontCheckSSL'][0] = 'Don\'t check SSL'; -$lang['dontCheckSSL'][1] = 'Activate this checkbox if you want to prevent checking of SSL-Certificates for this request. (Do this only if you know what this means!)'; +$lang['dontCheckSSL'][1] = 'Activate this option in order to prevent the checking of SSL certificates for this request. Use this with caution!'; $lang['sourceUrl'][0] = 'Url'; $lang['sourceUrl'][1] = 'Enter the URL to the file here.'; $lang['absolutePath'][0] = 'Absolute file path'; From bb5e1a1cf1a994d28a2cbb5fcc0ddd04b0646914 Mon Sep 17 00:00:00 2001 From: syzn <1295141+SyZn@users.noreply.github.com> Date: Mon, 26 Apr 2021 17:22:36 +0200 Subject: [PATCH 10/12] [Fix] Moved dontCheckSSL field after sourceUrl --- src/Resources/contao/dca/tl_entity_import_source.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Resources/contao/dca/tl_entity_import_source.php b/src/Resources/contao/dca/tl_entity_import_source.php index 8593340..923bf6c 100644 --- a/src/Resources/contao/dca/tl_entity_import_source.php +++ b/src/Resources/contao/dca/tl_entity_import_source.php @@ -81,7 +81,7 @@ // Subpalettes 'subpalettes' => [ - 'retrievalType_http' => 'dontCheckSSL,sourceUrl,httpMethod,httpAuth,fileType', + 'retrievalType_http' => 'sourceUrl,httpMethod,dontCheckSSL,httpAuth,fileType', 'retrievalType_contao_file_system' => 'fileSRC,fileType', 'retrievalType_absolute_path' => 'absolutePath', 'fileType_csv' => 'fileContent,csvHeaderRow,csvSkipEmptyLines,csvDelimiter,csvEnclosure,csvEscape,fieldMappingCopier,fieldMappingPresets,fieldMapping', @@ -257,7 +257,7 @@ 'label' => &$GLOBALS['TL_LANG']['tl_entity_import_source']['dontCheckSSL'], 'exclude' => true, 'inputType' => 'checkbox', - 'eval' => ['submitOnChange' => true, 'tl_class' => 'w50 m12'], + 'eval' => ['submitOnChange' => true, 'tl_class' => 'w50 m12 clr'], 'sql' => "char(1) NOT NULL default ''", ], 'httpMethod' => [ From a82639dd9f6c8fc743f4df273cac74c85a85fedf Mon Sep 17 00:00:00 2001 From: syzn <1295141+SyZn@users.noreply.github.com> Date: Mon, 26 Apr 2021 23:56:47 +0200 Subject: [PATCH 11/12] [Fix] httpAuth parameters fix naming --- src/Resources/contao/dca/tl_entity_import_source.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Resources/contao/dca/tl_entity_import_source.php b/src/Resources/contao/dca/tl_entity_import_source.php index 923bf6c..0bd1773 100644 --- a/src/Resources/contao/dca/tl_entity_import_source.php +++ b/src/Resources/contao/dca/tl_entity_import_source.php @@ -284,7 +284,7 @@ 'maxRowCount' => 1, 'sortable' => true, 'fields' => [ - 'name' => [ + 'username' => [ 'label' => &$GLOBALS['TL_LANG']['tl_entity_import_source']['httpAuth']['username'], 'exclude' => true, 'inputType' => 'text', @@ -292,7 +292,7 @@ 'groupStyle' => 'width: 49%', ], ], - 'value' => [ + 'password' => [ 'label' => &$GLOBALS['TL_LANG']['tl_entity_import_source']['httpAuth']['password'], 'exclude' => true, 'inputType' => 'text', From 65e7485c2a02add6f1ab35060468171ede5bf801 Mon Sep 17 00:00:00 2001 From: syzn <1295141+SyZn@users.noreply.github.com> Date: Tue, 27 Apr 2021 00:08:33 +0200 Subject: [PATCH 12/12] [Fix] Generalized auth and SSL verification --- src/Source/AbstractFileSource.php | 11 +++++++++-- src/Source/AbstractSource.php | 10 +++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Source/AbstractFileSource.php b/src/Source/AbstractFileSource.php index 5e28b74..6fbfe0f 100644 --- a/src/Source/AbstractFileSource.php +++ b/src/Source/AbstractFileSource.php @@ -77,10 +77,17 @@ public function getFileContent(bool $cache = false): string case EntityImportSourceContainer::RETRIEVAL_TYPE_HTTP: $auth = []; + $options = []; if (null !== $this->sourceModel->httpAuth) { $httpAuth = \Contao\StringUtil::deserialize($this->sourceModel->httpAuth, true); $auth = ['auth' => [$httpAuth['username'], $httpAuth['password']]]; + $options = $auth; + } + + // Check if SSL verification should be done + if($this->sourceModel->dontCheckSSL) { + $options['verify'] = false; } $event = $this->eventDispatcher->dispatch(BeforeAuthenticationEvent::NAME, new BeforeAuthenticationEvent($auth, $this->sourceModel)); @@ -91,14 +98,14 @@ public function getFileContent(bool $cache = false): string $content = $this->getValueFromRemoteCache($cacheKey); if (empty($content)) { - $this->storeValueToRemoteCache($this->sourceModel->sourceUrl, $cacheKey, $this->sourceModel->httpMethod, $event->getAuth(), ["verify" => (!$this->sourceModel->dontCheckSSL)]); + $this->storeValueToRemoteCache($this->sourceModel->sourceUrl, $cacheKey, $this->sourceModel->httpMethod, $options); $content = $this->getValueFromRemoteCache($cacheKey); } break; } - $result = $this->getContentFromUrl($this->sourceModel->httpMethod, $this->sourceModel->sourceUrl, $event->getAuth(), ["verify" => (!$this->sourceModel->dontCheckSSL))]); + $result = $this->getContentFromUrl($this->sourceModel->httpMethod, $this->sourceModel->sourceUrl, $options); $content = $result['result']; break; diff --git a/src/Source/AbstractSource.php b/src/Source/AbstractSource.php index 71db276..2ef54cb 100644 --- a/src/Source/AbstractSource.php +++ b/src/Source/AbstractSource.php @@ -100,12 +100,12 @@ protected function getMappedItemData(?array $element, array $mapping): array return $result; } - protected function getContentFromUrl(string $method, string $url, array $auth = [], array $HTTPClientOptions = []): array + protected function getContentFromUrl(string $method, string $url, array $options = []): array { - $client = new Client($HTTPClientOptions); + $client = new Client(); try { - $response = $client->request($method, \Contao\StringUtil::decodeEntities($url), $auth); + $response = $client->request($method, \Contao\StringUtil::decodeEntities($url), $options); } catch (RequestException $e) { if($e->hasResponse()){ return [ @@ -136,11 +136,11 @@ protected function deleteValueFromRemoteCache(string $cacheKey): string return $filesystemCache->deleteItem('entity-import-remote.'.$cacheKey); } - protected function storeValueToRemoteCache(string $url, string $cacheKey, string $method, array $auth = [], array $HTTPClientOptions = []) + protected function storeValueToRemoteCache(string $url, string $cacheKey, string $method, array $options = []) { $filesystemCache = $this->getFilesystemCache(); - $response = $this->getContentFromUrl($method, $url, $auth, $HTTPClientOptions); + $response = $this->getContentFromUrl($method, $url, $options); if (200 === $response['statusCode']) { $filesystemCache->set('entity-import-remote.'.$cacheKey, $response['result']);