Skip to content

Commit bdd9130

Browse files
KocKoc
committed
Enhancement: Make tables import asynchronous
Signed-off-by: Kostiantyn Miakshyn <molodchick@gmail.com>
1 parent 7bbc382 commit bdd9130

16 files changed

Lines changed: 491 additions & 290 deletions

‎appinfo/routes.php‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,13 +102,18 @@
102102

103103
// import
104104
['name' => 'import#previewImportTable', 'url' => '/import-preview/table/{tableId}', 'verb' => 'POST'],
105-
['name' => 'import#importInTable', 'url' => '/import/table/{tableId}', 'verb' => 'POST'],
105+
['name' => 'import#scheduleImportInTable', 'url' => '/import/table/{tableId}/jobs', 'verb' => 'POST'],
106+
['name' => 'import#scheduleImportInView', 'url' => '/import/view/{viewId}/jobs', 'verb' => 'POST'],
106107
['name' => 'import#previewImportView', 'url' => '/import-preview/view/{viewId}', 'verb' => 'POST'],
107-
['name' => 'import#importInView', 'url' => '/import/view/{viewId}', 'verb' => 'POST'],
108108
['name' => 'import#previewUploadImportTable', 'url' => '/importupload-preview/table/{tableId}', 'verb' => 'POST'],
109-
['name' => 'import#importUploadInTable', 'url' => '/importupload/table/{tableId}', 'verb' => 'POST'],
109+
['name' => 'import#scheduleImportUploadInTable', 'url' => '/importupload/table/{tableId}/jobs', 'verb' => 'POST'],
110110
['name' => 'import#previewUploadImportView', 'url' => '/importupload-preview/view/{viewId}', 'verb' => 'POST'],
111+
['name' => 'import#scheduleImportUploadInView', 'url' => '/importupload/view/{viewId}/jobs', 'verb' => 'POST'],
112+
// deprecated endpoints
113+
['name' => 'import#importUploadInTable', 'url' => '/importupload/table/{tableId}', 'verb' => 'POST'],
111114
['name' => 'import#importUploadInView', 'url' => '/importupload/view/{viewId}', 'verb' => 'POST'],
115+
['name' => 'import#importInTable', 'url' => '/import/table/{tableId}', 'verb' => 'POST'],
116+
['name' => 'import#importInView', 'url' => '/import/view/{viewId}', 'verb' => 'POST'],
112117

113118
// search
114119
['name' => 'search#all', 'url' => '/search/all', 'verb' => 'GET'],

‎cypress/e2e/tables-import.cy.js‎

Lines changed: 16 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -27,20 +27,12 @@ describe('Import csv', () => {
2727
cy.get('.file-picker button span').contains('Import').click()
2828
cy.get('.modal__content .import-filename', { timeout: 5000 }).should('be.visible')
2929

30-
cy.intercept({ method: 'POST', url: '**/apps/tables/import-preview/**' }).as('importPreviewPath')
3130
cy.get('.modal__content button').contains('Preview').click()
32-
cy.wait('@importPreviewPath')
33-
cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 4)
31+
cy.get('.file_import__preview tbody tr').should('have.length', 4)
3432

35-
cy.intercept({ method: 'POST', url: '**/apps/tables/import/table/*' }).as('importUploadReq')
33+
cy.intercept({ method: 'POST', url: '**/apps/tables/import/table/*/jobs' }).as('importUploadReq')
3634
cy.get('.modal__content button').contains('Import').click()
37-
cy.wait('@importUploadReq')
38-
cy.get('[data-cy="importResultColumnsFound"]').should('contain.text', '4')
39-
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '4')
40-
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
41-
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
42-
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
43-
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
35+
cy.wait('@importUploadReq').its('response.statusCode').should('equal', 200)
4436
})
4537

4638
it('Import csv from device', () => {
@@ -49,20 +41,12 @@ describe('Import csv', () => {
4941
cy.get('.modal__content button').contains('Upload from device').click()
5042
cy.get('input[type="file"]').selectFile('cypress/fixtures/test-import.csv', { force: true })
5143

52-
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload-preview/**' }).as('importPreviewUpload')
5344
cy.get('.modal__content button').contains('Preview').click()
54-
cy.wait('@importPreviewUpload')
5545
cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 4)
5646

57-
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload/table/*' }).as('importUploadReq')
47+
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload/table/*/jobs' }).as('importUploadReq')
5848
cy.get('.modal__content button').contains('Import').click()
59-
cy.wait('@importUploadReq')
60-
cy.get('[data-cy="importResultColumnsFound"]', { timeout: 20000 }).should('contain.text', '4')
61-
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '4')
62-
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
63-
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
64-
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
65-
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
49+
cy.wait('@importUploadReq').its('response.statusCode').should('equal', 200)
6650
})
6751

6852
it('Import csv from device with updating of existent files', () => {
@@ -84,99 +68,57 @@ describe('Import csv', () => {
8468
cy.get('.modal__content button').contains('Upload from device').click()
8569
cy.get('input[type="file"]').selectFile('cypress/fixtures/test-import-update.csv', { force: true })
8670

87-
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload-preview/**' }).as('importPreviewUpdate')
8871
cy.get('.modal__content button').contains('Preview').click()
89-
cy.wait('@importPreviewUpdate')
9072
cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 3)
9173

92-
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload/table/*' }).as('importUploadReq')
74+
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload/table/*/jobs' }).as('importUploadReq')
9375
cy.get('.modal__content button').contains('Import').click()
94-
cy.wait('@importUploadReq')
95-
cy.get('[data-cy="importResultColumnsFound"]', { timeout: 20000 }).should('contain.text', '2')
96-
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '3')
97-
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
98-
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '0')
99-
cy.get('[data-cy="importResultRowsUpdated"]').should('contain.text', '1')
100-
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
101-
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
76+
cy.wait('@importUploadReq').its('response.statusCode').should('equal', 200)
10277
})
10378

10479
})
10580

10681
describe('Import csv from Files file action', () => {
107-
const rowActionsButton = '[data-cy-files-list-row-name="test-import.csv"] [data-cy-files-list-row-actions] .action-item button'
108-
const importToTablesAction = '[data-cy-files-list-row-action="import-to-tables"]'
109-
110-
const openImportToTablesAction = (retries = 8) => {
111-
cy.get(rowActionsButton).click()
112-
cy.get('body').then($body => {
113-
if ($body.find(importToTablesAction).length > 0) {
114-
cy.get(importToTablesAction).first().click()
115-
return
116-
}
117-
118-
if (retries <= 0) {
119-
throw new Error('Import to Tables action did not become available in file actions menu')
120-
}
121-
122-
cy.get('body').click(0, 0)
123-
cy.wait(1000)
124-
openImportToTablesAction(retries - 1)
125-
})
126-
}
12782

128-
before(function() {
83+
before(function () {
12984
cy.createRandomUser().then(user => {
13085
localUser = user
13186
cy.login(localUser)
13287
cy.uploadFile('test-import.csv', 'text/csv')
13388
})
13489
})
13590

136-
beforeEach(function() {
91+
beforeEach(function () {
13792
cy.login(localUser)
138-
cy.visit('apps/files/files', { timeout: 120000 })
139-
cy.get('[data-cy-files-list-row-name="test-import.csv"]', { timeout: 120000 }).should('be.visible')
93+
cy.visit('apps/files/files')
14094
})
14195

14296
it('Import to new table', () => {
143-
openImportToTablesAction()
97+
cy.get('[data-cy-files-list-row-name="test-import.csv"] [data-cy-files-list-row-actions] .action-item button').click()
98+
cy.get('[data-cy-files-list-row-action="import-to-tables"]').click()
14499

145100
cy.intercept({
146101
method: 'POST',
147-
url: '**/apps/tables/import/table/*',
102+
url: '**/apps/tables/import/table/*/jobs',
148103
}).as('importNewTableReq')
149104
cy.get('[data-cy="fileActionImportButton"]').click({ force: true })
150105
cy.wait('@importNewTableReq').its('response.statusCode').should('equal', 200)
151-
152-
cy.get('[data-cy="importResultColumnsFound"]').should('contain.text', '4')
153-
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '0')
154-
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '4')
155-
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
156-
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
157-
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
158106
})
159107

160108
it('Import to existing table', () => {
161-
openImportToTablesAction()
109+
cy.get('[data-cy-files-list-row-name="test-import.csv"] [data-cy-files-list-row-actions] .action-item button').click()
110+
cy.get('[data-cy-files-list-row-action="import-to-tables"]').click()
162111

163112
cy.get('.modal__content [data-cy="importAsNewTableSwitch"] input').uncheck({ force: true })
164113
cy.get('[data-cy="selectExistingTableDropdown"]').type('Welcome to Nextcloud Tables!')
165114
cy.get('.name-parts').click()
166115

167116
cy.intercept({
168117
method: 'POST',
169-
url: '**/apps/tables/import/table/*',
118+
url: '**/apps/tables/import/table/*/jobs',
170119
}).as('importExistingTableReq')
171120
cy.get('[data-cy="fileActionImportButton"]').click({ force: true })
172121
cy.wait('@importExistingTableReq').its('response.statusCode').should('equal', 200)
173-
174-
cy.get('[data-cy="importResultColumnsFound"]').should('contain.text', '4')
175-
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '4')
176-
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
177-
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
178-
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
179-
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
180122
})
181123

182124
})

‎cypress/e2e/tables-table.cy.js‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,15 +51,9 @@ describe('Manage a table', () => {
5151
cy.get('.file-picker button span').contains('Import').click()
5252
cy.get('.modal__content button').contains('Preview').click()
5353
cy.get('.file_import__preview tbody tr').should('have.length', 4)
54-
cy.intercept({ method: 'POST', url: '**/apps/tables/import/table/*' }).as('importUploadReq')
54+
cy.intercept({ method: 'POST', url: '**/apps/tables/import/table/*/jobs'}).as('importUploadReq')
5555
cy.get('.modal__content button').contains('Import').scrollIntoView().click()
56-
cy.wait('@importUploadReq')
57-
cy.get('[data-cy="importResultColumnsFound"]').should('contain.text', '4')
58-
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '0')
59-
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '4')
60-
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
61-
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
62-
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
56+
cy.wait('@importUploadReq').its('response.statusCode').should('equal', 200)
6357
})
6458

6559
it('Update description', () => {

‎lib/Activity/ActivityManager.php‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ class ActivityManager {
3535
public const SUBJECT_ROW_UPDATE = 'row_update';
3636
public const SUBJECT_ROW_DELETE = 'row_delete';
3737

38+
public const SUBJECT_IMPORT_FINISHED = 'import_finished';
39+
3840
public function __construct(
3941
private readonly IManager $manager,
4042
private readonly IFactory $l10nFactory,
@@ -135,9 +137,11 @@ private function createEvent($objectType, $object, $subject, $additionalParams =
135137
case self::SUBJECT_ROW_DELETE:
136138
$subjectParams['row'] = $object;
137139
break;
138-
default:
139-
throw new \Exception('Unknown subject for activity.');
140+
case self::SUBJECT_IMPORT_FINISHED:
141+
$subjectParams['importStats'] = $additionalParams['importStats'] ?? null;
140142
break;
143+
default:
144+
throw new \Exception(sprintf('Unknown subject "%s" for activity.', $subject));
141145
}
142146

143147
if ($subject === self::SUBJECT_ROW_UPDATE) {
@@ -200,7 +204,7 @@ private function sendToUsers(IEvent $event, $object) {
200204
}
201205
}
202206

203-
public function getActivityFormat($language, $subjectIdentifier, $subjectParams = [], $ownActivity = false) {
207+
public function getActivitySubject($language, $subjectIdentifier, $subjectParams = [], $ownActivity = false) {
204208
$subject = '';
205209
$l = $this->l10nFactory->get(Application::APP_ID, $language);
206210

@@ -248,10 +252,34 @@ public function getActivityFormat($language, $subjectIdentifier, $subjectParams
248252
case self::SUBJECT_ROW_DELETE:
249253
$subject = $ownActivity ? $l->t('You have deleted the row {row} in table {table}') : $l->t('{user} has deleted the row {row} in table {table}');
250254
break;
255+
case self::SUBJECT_IMPORT_FINISHED:
256+
$subject = $ownActivity ? $l->t('You have imported file to table {table}') : $l->t('{user} has imported file to table {table}');
257+
break;
251258
default:
252259
break;
253260
}
254261

255262
return $subject;
256263
}
264+
265+
public function getActivityMessage($language, $subjectIdentifier) {
266+
$l = $this->l10nFactory->get(Application::APP_ID, $language);
267+
268+
switch ($subjectIdentifier) {
269+
case self::SUBJECT_IMPORT_FINISHED:
270+
$lines = [
271+
$l->t('Found columns: {foundColumnsCount}'),
272+
$l->t('Matching columns: {matchingColumnsCount}'),
273+
$l->t('Created columns: {createdColumnsCount}'),
274+
$l->t('Inserted rows: {insertedRowsCount}'),
275+
$l->t('Updated rows: {updatedRowsCount}'),
276+
$l->t('Value parsing errors: {errorsParsingCount}'),
277+
$l->t('Row creation errors: {errorsCount}'),
278+
];
279+
return implode("\n", $lines);
280+
281+
default:
282+
return null;
283+
}
284+
}
257285
}

0 commit comments

Comments
 (0)