Skip to content

Commit 796d972

Browse files
a4bluea4blue
committed
Added a Setting to set the Model Download Folder
Signed-off-by: Alexander Ratajczak <a4blue@hotmail.de>
1 parent c29bd8a commit 796d972

7 files changed

Lines changed: 141 additions & 17 deletions

File tree

‎appinfo/info.xml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,9 @@ The app does not send any sensitive data to cloud providers or similar services.
102102
<post-migration>
103103
<step>OCA\Recognize\Migration\InstallDeps</step>
104104
</post-migration>
105+
<post-migration>
106+
<step>OCA\Recognize\Migration\MoveDefaultModelFolder</step>
107+
</post-migration>
105108
<live-migration>
106109
<step>OCA\Recognize\Migration\RemoveDuplicateFaceDetections</step>
107110
</live-migration>

‎lib/Migration/MoveDefaultModelFolder.php‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* @copyright Copyright (c) 2020, Joas Schilling <coding@schilljs.com>
6+
* @copyright Copyright (c) 2021, Marcel Klehr <mklehr@gmx.net>
7+
*
8+
* @author Joas Schilling <coding@schilljs.com>
9+
*
10+
* @license GNU AGPL version 3 or any later version
11+
*
12+
* This program is free software: you can redistribute it and/or modify
13+
* it under the terms of the GNU Affero General Public License as
14+
* published by the Free Software Foundation, either version 3 of the
15+
* License, or (at your option) any later version.
16+
*
17+
* This program is distributed in the hope that it will be useful,
18+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
19+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20+
* GNU Affero General Public License for more details.
21+
*
22+
* You should have received a copy of the GNU Affero General Public License
23+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
24+
*
25+
*/
26+
namespace OCA\Recognize\Migration;
27+
28+
use OCA\Recognize\Service\SettingsService;
29+
use OCP\Migration\IOutput;
30+
use OCP\Migration\IRepairStep;
31+
use Psr\Log\LoggerInterface;
32+
use function Safe\rename;
33+
use function Safe\scandir;
34+
35+
final class MoveDefaultModelFolder implements IRepairStep {
36+
37+
public function __construct(
38+
private SettingsService $settingsService,
39+
private LoggerInterface $logger,
40+
) {
41+
}
42+
43+
public function getName(): string {
44+
return 'Try to move the default Model Folder';
45+
}
46+
47+
public function run(IOutput $output): void {
48+
$oldModelTargetPath = __DIR__ . '/../../models';
49+
$oldModelArchivePath = __DIR__ . '/../../models.tar.gz';
50+
$newPath = $this->settingsService->getSetting('models_target_path');
51+
$newModelTargetPath = $newPath . '/models';
52+
$newModelArchivePath = $newPath . '/models.tar.gz';
53+
54+
if (is_dir($oldModelTargetPath)) {
55+
$filesToMove = scandir($oldModelTargetPath);
56+
$filesToMove = array_filter($filesToMove, fn ($value) => $value !== '.' && $value === '..');
57+
$filesToMove = array_map(fn ($value) => $oldModelTargetPath.'/'.$value, $filesToMove);
58+
mkdir($newModelTargetPath);
59+
foreach ($filesToMove as $file) {
60+
rename($file, $newModelTargetPath.'/'.basename($file));
61+
}
62+
}
63+
64+
if (is_file($oldModelArchivePath)) {
65+
rename($oldModelArchivePath, $newModelArchivePath);
66+
}
67+
}
68+
}

‎lib/Service/DownloadModelsService.php‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,24 @@
1616
final class DownloadModelsService {
1717
private IClientService $clientService;
1818
private bool $isCLI;
19+
private SettingsService $settingsService;
1920

20-
public function __construct(IClientService $clientService, bool $isCLI) {
21+
public function __construct(IClientService $clientService, bool $isCLI, SettingsService $settingsService) {
2122
$this->clientService = $clientService;
2223
$this->isCLI = $isCLI;
24+
$this->settingsService = $settingsService;
2325
}
2426

2527
/**
2628
* @return void
2729
* @throws \Exception
2830
*/
2931
public function download() : void {
30-
$targetPath = __DIR__ . '/../../models';
31-
if (file_exists($targetPath)) {
32+
$targetPath = $this->settingsService->getSetting('models_target_path');
33+
$modelPath = $targetPath . '/models';
34+
if (file_exists($modelPath)) {
3235
// remove models directory
33-
$it = new RecursiveDirectoryIterator($targetPath, FilesystemIterator::SKIP_DOTS);
36+
$it = new RecursiveDirectoryIterator($modelPath, FilesystemIterator::SKIP_DOTS);
3437
$files = new RecursiveIteratorIterator($it,
3538
RecursiveIteratorIterator::CHILD_FIRST);
3639
foreach ($files as $file) {
@@ -40,11 +43,11 @@ public function download() : void {
4043
unlink($file->getRealPath());
4144
}
4245
}
43-
rmdir($targetPath);
46+
rmdir($modelPath);
4447
}
4548

4649
$archiveUrl = $this->getArchiveUrl($this->getNeededArchiveRef());
47-
$archivePath = __DIR__ . '/../../models.tar.gz';
50+
$archivePath = $targetPath . '/models.tar.gz';
4851
$timeout = $this->isCLI ? 0 : 480;
4952
$this->clientService->newClient()->get($archiveUrl, ['sink' => $archivePath, 'timeout' => $timeout]);
5053
$tarManager = new TAR($archivePath);

‎lib/Service/SettingsService.php‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ final class SettingsService {
6161
'nice_value' => '0',
6262
'concurrency.enabled' => 'false',
6363
'ffmpeg_binary' => '',
64+
'models_target_path' => '../../models_cache',
6465
];
6566

6667
/** @var array<string,string> */
@@ -94,7 +95,13 @@ final class SettingsService {
9495
'landmarks.batchSize',
9596
'movinet.batchSize',
9697
'musicnn.batchSize',
97-
'concurrency.enabled'
98+
'concurrency.enabled',
99+
'models_target_path',
100+
'models_archive_file',
101+
];
102+
103+
private const PATH_SETTINGS = [
104+
'models_target_path',
98105
];
99106

100107
private IAppConfig $config;
@@ -121,6 +128,14 @@ public function getSetting(string $key): string {
121128
if (in_array($key, self::LAZY_SETTINGS, true)) {
122129
$lazy = true;
123130
}
131+
132+
if (in_array($key, self::PATH_SETTINGS, true)) {
133+
$path = $this->config->getAppValueString($key, self::DEFAULTS[$key], lazy: $lazy);
134+
if (!$this->isPathAbsolute($path)) {
135+
$path = __DIR__ .'/'. $path;
136+
}
137+
return $path;
138+
}
124139
return $this->config->getAppValueString($key, self::DEFAULTS[$key], lazy: $lazy);
125140
}
126141

@@ -182,6 +197,9 @@ public function setSetting(string $key, string $value): void {
182197
if (in_array($key, self::LAZY_SETTINGS, true)) {
183198
$lazy = true;
184199
}
200+
if (in_array($key, self::PATH_SETTINGS) && $value === '') {
201+
$value = self::DEFAULTS[$key];
202+
}
185203
$this->config->setAppValueString($key, $value, lazy: $lazy);
186204
}
187205

@@ -195,4 +213,15 @@ public function getAll(): array {
195213
}
196214
return $settings;
197215
}
216+
217+
private function isPathAbsolute(string $path): bool {
218+
if ($path === '') {
219+
return false;
220+
}
221+
if ($path[0] === '/') {
222+
return true;
223+
}
224+
225+
return false;
226+
}
198227
}

‎lib/Settings/AdminSettings.php‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,11 @@ public function getForm(): TemplateResponse {
2828
$settings = $this->settingsService->getAll();
2929
$this->initialState->provideInitialState('settings', $settings);
3030

31-
$modelsPath = __DIR__ . '/../../models';
32-
$modelsDownloaded = file_exists($modelsPath);
31+
$targetPath = $this->settingsService->getSetting('models_target_path');
32+
$modelsDownloaded = file_exists($targetPath .' /models');
33+
$modelsTargetPathWritable = is_writable($targetPath);
3334
$this->initialState->provideInitialState('modelsDownloaded', $modelsDownloaded);
35+
$this->initialState->provideInitialState('modelsTargetPathWritable', $modelsTargetPathWritable);
3436

3537
$tagsEnabled = $this->appManager->isEnabledForAnyone('systemtags');
3638
$this->initialState->provideInitialState('tagsEnabled', $tagsEnabled);

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/components/ViewAdmin.vue‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,23 @@
3838
</p>
3939
</template>
4040
</NcSettingsSection>
41+
<NcSettingsSection :name="t('recognize', 'Model Setting')">
42+
<NcNoteCard v-if="modelsTargetPathWritable" show-alert type="success">
43+
{{ t('recognize', 'The Model Target Path is Writable') }}
44+
</NcNoteCard>
45+
<NcNoteCard v-else-if="!modelsTargetPathWritable" show-alert type="warning">
46+
{{ t('recognize', 'Model Target Path is not Writable') }}
47+
</NcNoteCard>
48+
<p>
49+
<NcTextField v-model="settings['models_target_path']"
50+
:label="t('recognize', 'Path Model Folder')"
51+
:label-visible="true"
52+
@update:model-value="onChange" />
53+
</p>
54+
<p>
55+
{{ t('recognize', 'Changing this Value will require you to redownload the Models. Also the models in the old Directory need to be deleted manually') }}
56+
</p>
57+
</NcSettingsSection>
4158
<NcSettingsSection :name="t('recognize', 'Classifier backend')">
4259
<NcNoteCard v-if="recognizeBackendInstalled" type="success">
4360
{{ t('recognize', 'The recognize_backend ExApp is installed; TaskProcessing mode is recommended.') }}
@@ -81,8 +98,8 @@
8198
{{ t('recognize', 'Enable face recognition (groups photos by faces that appear in them; UI is in the photos app)') }}
8299
</NcCheckboxRadioSwitch>
83100
<NcTextField v-if="!settings['taskprocessing.enabled']"
84-
:disabled="!settings['faces.enabled']"
85101
v-model="settings['faces.batchSize']"
102+
:disabled="!settings['faces.enabled']"
86103
:label-visible="true"
87104
:label="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~500 or more, in WASM mode ~50 is recommended)')"
88105
:title="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~500 or more, in WASM mode ~50 is recommended)')"
@@ -130,8 +147,8 @@
130147
{{ t('recognize', 'Enable object recognition (e.g. food, vehicles, landscapes)') }}
131148
</NcCheckboxRadioSwitch>
132149
<NcTextField v-if="!settings['taskprocessing.enabled']"
133-
:disabled="!settings['imagenet.enabled']"
134150
v-model="settings['imagenet.batchSize']"
151+
:disabled="!settings['imagenet.enabled']"
135152
:label-visible="true"
136153
:label="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~100 or more, in WASM mode ~20 is recommended)')"
137154
:title="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~100 or more, in WASM mode ~20 is recommended)')"
@@ -146,8 +163,8 @@
146163
{{ t('recognize', 'Enable landmark recognition (e.g. Eiffel Tower, Golden Gate Bridge)') }}
147164
</NcCheckboxRadioSwitch>
148165
<NcTextField v-if="!settings['taskprocessing.enabled']"
149-
:disabled="!settings['imagenet.enabled'] || !settings['landmarks.enabled']"
150166
v-model="settings['landmarks.batchSize']"
167+
:disabled="!settings['imagenet.enabled'] || !settings['landmarks.enabled']"
151168
:label-visible="true"
152169
:label="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~100 or more, in WASM mode ~20 is recommended)')"
153170
:title="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~100 or more, in WASM mode ~20 is recommended)')"
@@ -177,8 +194,8 @@
177194
{{ t('recognize', 'Enable music genre recognition (e.g. pop, rock, folk, metal, new age)') }}
178195
</NcCheckboxRadioSwitch>
179196
<NcTextField v-if="!settings['taskprocessing.enabled']"
180-
:disabled="!settings['musicnn.enabled']"
181197
v-model="settings['musicnn.batchSize']"
198+
:disabled="!settings['musicnn.enabled']"
182199
:label-visible="true"
183200
:label="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~100 or more, in WASM mode ~20 is recommended)')"
184201
:title="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~100 or more, in WASM mode ~20 is recommended)')"
@@ -211,8 +228,8 @@
211228
{{ t('recognize', 'Enable human action recognition (e.g. arm wrestling, dribbling basketball, hula hooping)') }}
212229
</NcCheckboxRadioSwitch>
213230
<NcTextField v-if="!settings['taskprocessing.enabled']"
214-
:disabled="!settings['movinet.enabled']"
215231
v-model="settings['movinet.batchSize']"
232+
:disabled="!settings['movinet.enabled']"
216233
:label-visible="true"
217234
:label="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~20 or more, in WASM mode ~5 is recommended)')"
218235
:title="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~20 or more, in WASM mode ~5 is recommended)')"
@@ -445,7 +462,7 @@ const TASK_PROCESSING_TASK_TYPES = {
445462
musicnn: 'recognize:audio:classification',
446463
}
447464
448-
const SETTINGS = ['tensorflow.cores', 'tensorflow.gpu', 'tensorflow.purejs', 'imagenet.enabled', 'landmarks.enabled', 'faces.enabled', 'musicnn.enabled', 'movinet.enabled', 'node_binary', 'ffmpeg_binary', 'faces.status', 'imagenet.status', 'landmarks.status', 'movinet.status', 'musicnn.status', 'faces.lastFile', 'imagenet.lastFile', 'landmarks.lastFile', 'movinet.lastFile', 'musicnn.lastFile', 'faces.batchSize', 'imagenet.batchSize', 'landmarks.batchSize', 'movinet.batchSize', 'musicnn.batchSize', 'clusterFaces.status', 'clusterFaces.lastRun', 'nice_binary', 'nice_value', 'concurrency.enabled', 'taskprocessing.enabled']
465+
const SETTINGS = ['tensorflow.cores', 'tensorflow.gpu', 'tensorflow.purejs', 'imagenet.enabled', 'landmarks.enabled', 'faces.enabled', 'musicnn.enabled', 'movinet.enabled', 'node_binary', 'ffmpeg_binary', 'faces.status', 'imagenet.status', 'landmarks.status', 'movinet.status', 'musicnn.status', 'faces.lastFile', 'imagenet.lastFile', 'landmarks.lastFile', 'movinet.lastFile', 'musicnn.lastFile', 'faces.batchSize', 'imagenet.batchSize', 'landmarks.batchSize', 'movinet.batchSize', 'musicnn.batchSize', 'clusterFaces.status', 'clusterFaces.lastRun', 'nice_binary', 'nice_value', 'concurrency.enabled', 'taskprocessing.enabled', 'models_target_path', 'models_archive_file']
449466
450467
const BOOLEAN_SETTINGS = ['tensorflow.gpu', 'tensorflow.purejs', 'imagenet.enabled', 'landmarks.enabled', 'faces.enabled', 'musicnn.enabled', 'movinet.enabled', 'faces.status', 'imagenet.status', 'landmarks.status', 'movinet.status', 'musicnn.status', 'faces.lastFile', 'imagenet.lastFile', 'landmarks.lastFile', 'movinet.lastFile', 'musicnn.lastFile', 'clusterFaces.status', 'concurrency.enabled', 'taskprocessing.enabled']
451468
@@ -487,6 +504,7 @@ export default {
487504
musicnnTpStats: null,
488505
tagsEnabled: null,
489506
recognizeBackendInstalled: false,
507+
modelsTargetPathWritable: null,
490508
}
491509
},
492510
@@ -523,6 +541,7 @@ export default {
523541
},
524542
async created() {
525543
this.modelsDownloaded = loadState('recognize', 'modelsDownloaded')
544+
this.modelsTargetPathWritable = loadState('recognize', 'modelsTargetPathWritable')
526545
this.tagsEnabled = loadState('recognize', 'tagsEnabled')
527546
this.recognizeBackendInstalled = loadState('recognize', 'recognizeBackendInstalled', false)
528547
this.getCount()

0 commit comments

Comments
 (0)