Skip to content

Commit fa1c6f4

Browse files
KocKoc
committed
Enhancement: Make tables import asynchronous (code review fixes)
Signed-off-by: Kostiantyn Miakshyn <molodchick@gmail.com>
1 parent bdd9130 commit fa1c6f4

12 files changed

Lines changed: 1531 additions & 92 deletions

File tree

‎REUSE.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ SPDX-FileCopyrightText = "2017 Nextcloud GmbH and Nextcloud contributors"
6666
SPDX-License-Identifier = "AGPL-3.0-or-later"
6767

6868
[[annotations]]
69-
path = ["cypress/styleguide/assets/img/filetypes/font.svg", "cypress/styleguide/assets/img/actions/star-rounded.svg", "cypress/fixtures/example.json", "cypress/fixtures/NC_server_test.pdf", "cypress/fixtures/photo-test-1.jpeg", "cypress/fixtures/test-import.csv", "playwright/fixtures/example.json", "playwright/fixtures/NC_server_test.pdf", "playwright/fixtures/photo-test-1.jpeg", "playwright/fixtures/test-import.csv", "docs/permissions-matrix.xlsx", "tests/stub.phpstub", "tests/integration/composer.json", "tests/integration/composer.lock", "tests/integration/base-query-count.txt"]
69+
path = ["cypress/styleguide/assets/img/filetypes/font.svg", "cypress/styleguide/assets/img/actions/star-rounded.svg", "cypress/fixtures/example.json", "cypress/fixtures/NC_server_test.pdf", "cypress/fixtures/photo-test-1.jpeg", "cypress/fixtures/test-import-large.csv", "cypress/fixtures/test-import-small.csv", "playwright/fixtures/example.json", "playwright/fixtures/NC_server_test.pdf", "playwright/fixtures/photo-test-1.jpeg", "playwright/fixtures/test-import.csv", "docs/permissions-matrix.xlsx", "tests/stub.phpstub", "tests/integration/composer.json", "tests/integration/composer.lock", "tests/integration/base-query-count.txt"]
7070
precedence = "aggregate"
7171
SPDX-FileCopyrightText = "2023 Nextcloud GmbH and Nextcloud contributors"
7272
SPDX-License-Identifier = "AGPL-3.0-or-later"

‎appinfo/routes.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,13 +102,13 @@
102102

103103
// import
104104
['name' => 'import#previewImportTable', 'url' => '/import-preview/table/{tableId}', 'verb' => 'POST'],
105-
['name' => 'import#scheduleImportInTable', 'url' => '/import/table/{tableId}/jobs', 'verb' => 'POST'],
106-
['name' => 'import#scheduleImportInView', 'url' => '/import/view/{viewId}/jobs', 'verb' => 'POST'],
105+
['name' => 'import#importV2InTable', 'url' => '/v2/import/table/{tableId}', 'verb' => 'POST'],
106+
['name' => 'import#importV2InView', 'url' => '/v2/import/view/{viewId}', 'verb' => 'POST'],
107107
['name' => 'import#previewImportView', 'url' => '/import-preview/view/{viewId}', 'verb' => 'POST'],
108108
['name' => 'import#previewUploadImportTable', 'url' => '/importupload-preview/table/{tableId}', 'verb' => 'POST'],
109-
['name' => 'import#scheduleImportUploadInTable', 'url' => '/importupload/table/{tableId}/jobs', 'verb' => 'POST'],
110109
['name' => 'import#previewUploadImportView', 'url' => '/importupload-preview/view/{viewId}', 'verb' => 'POST'],
111-
['name' => 'import#scheduleImportUploadInView', 'url' => '/importupload/view/{viewId}/jobs', 'verb' => 'POST'],
110+
['name' => 'import#importV2UploadInTable', 'url' => '/v2/importupload/table/{tableId}', 'verb' => 'POST'],
111+
['name' => 'import#importV2UploadInView', 'url' => '/v2/importupload/view/{viewId}', 'verb' => 'POST'],
112112
// deprecated endpoints
113113
['name' => 'import#importUploadInTable', 'url' => '/importupload/table/{tableId}', 'verb' => 'POST'],
114114
['name' => 'import#importUploadInView', 'url' => '/importupload/view/{viewId}', 'verb' => 'POST'],

‎cypress/e2e/tables-import.cy.js‎

Lines changed: 127 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ describe('Import csv', () => {
1010
cy.createRandomUser().then(user => {
1111
localUser = user
1212
cy.login(localUser)
13-
cy.uploadFile('test-import.csv', 'text/csv')
13+
cy.uploadFile('test-import-small.csv', 'text/csv')
14+
cy.uploadFile('test-import-large.csv', 'text/csv')
1415
})
1516
})
1617

@@ -19,32 +20,82 @@ describe('Import csv', () => {
1920
cy.visit('apps/tables')
2021
})
2122

22-
it('Import csv from Files', () => {
23+
it('Import small csv from Files', () => {
2324
cy.loadTable('Welcome to Nextcloud Tables!')
2425
cy.clickOnTableThreeDotMenu('Import')
2526
cy.get('.modal__content button').contains('Select from Files').click()
26-
cy.get('.file-picker__files').contains('test-import').click()
27+
cy.get('.file-picker__files').contains('test-import-small').click()
2728
cy.get('.file-picker button span').contains('Import').click()
2829
cy.get('.modal__content .import-filename', { timeout: 5000 }).should('be.visible')
2930

31+
cy.intercept({ method: 'POST', url: '**/apps/tables/import-preview/**' }).as('importPreviewPath')
3032
cy.get('.modal__content button').contains('Preview').click()
31-
cy.get('.file_import__preview tbody tr').should('have.length', 4)
33+
cy.wait('@importPreviewPath')
34+
cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 4)
3235

33-
cy.intercept({ method: 'POST', url: '**/apps/tables/import/table/*/jobs' }).as('importUploadReq')
36+
cy.intercept({ method: 'POST', url: '**/apps/tables/v2/import/table/*' }).as('importUploadReq')
37+
cy.get('.modal__content button').contains('Import').click()
38+
cy.wait('@importUploadReq')
39+
cy.get('[data-cy="importResultColumnsFound"]').should('contain.text', '4')
40+
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '4')
41+
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
42+
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
43+
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
44+
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
45+
})
46+
47+
it('Import large csv from Files', () => {
48+
cy.loadTable('Welcome to Nextcloud Tables!')
49+
cy.clickOnTableThreeDotMenu('Import')
50+
cy.get('.modal__content button').contains('Select from Files').click()
51+
cy.get('.file-picker__files').contains('test-import-large').click()
52+
cy.get('.file-picker button span').contains('Import').click()
53+
cy.get('.modal__content .import-filename', { timeout: 5000 }).should('be.visible')
54+
55+
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload-preview/**' }).as('importPreviewPath')
56+
cy.get('.modal__content button').contains('Preview').click()
57+
cy.wait('@importPreviewPath')
58+
cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 4)
59+
60+
cy.intercept({ method: 'POST', url: '**/apps/tables/v2/import/table/*' }).as('importUploadReq')
3461
cy.get('.modal__content button').contains('Import').click()
3562
cy.wait('@importUploadReq').its('response.statusCode').should('equal', 200)
3663
})
3764

38-
it('Import csv from device', () => {
65+
it('Import small csv from device', () => {
3966
cy.loadTable('Welcome to Nextcloud Tables!')
4067
cy.clickOnTableThreeDotMenu('Import')
4168
cy.get('.modal__content button').contains('Upload from device').click()
42-
cy.get('input[type="file"]').selectFile('cypress/fixtures/test-import.csv', { force: true })
69+
cy.get('input[type="file"]').selectFile('cypress/fixtures/test-import-small.csv', { force: true })
4370

71+
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload-preview/**' }).as('importPreviewUpload')
4472
cy.get('.modal__content button').contains('Preview').click()
73+
cy.wait('@importPreviewUpload')
4574
cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 4)
4675

47-
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload/table/*/jobs' }).as('importUploadReq')
76+
cy.intercept({ method: 'POST', url: '**/apps/tables/v2/importupload/table/*' }).as('importUploadReq')
77+
cy.get('.modal__content button').contains('Import').click()
78+
cy.wait('@importUploadReq')
79+
cy.get('[data-cy="importResultColumnsFound"]', { timeout: 20000 }).should('contain.text', '4')
80+
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '4')
81+
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
82+
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
83+
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
84+
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
85+
})
86+
87+
it('Import large csv from device', () => {
88+
cy.loadTable('Welcome to Nextcloud Tables!')
89+
cy.clickOnTableThreeDotMenu('Import')
90+
cy.get('.modal__content button').contains('Upload from device').click()
91+
cy.get('input[type="file"]').selectFile('cypress/fixtures/test-import-large.csv', { force: true })
92+
93+
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload-preview/**' }).as('importPreviewUpload')
94+
cy.get('.modal__content button').contains('Preview').click()
95+
cy.wait('@importPreviewUpload')
96+
cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 4)
97+
98+
cy.intercept({ method: 'POST', url: '**/apps/tables/v2/importupload/table/*' }).as('importUploadReq')
4899
cy.get('.modal__content button').contains('Import').click()
49100
cy.wait('@importUploadReq').its('response.statusCode').should('equal', 200)
50101
})
@@ -68,57 +119,111 @@ describe('Import csv', () => {
68119
cy.get('.modal__content button').contains('Upload from device').click()
69120
cy.get('input[type="file"]').selectFile('cypress/fixtures/test-import-update.csv', { force: true })
70121

122+
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload-preview/**' }).as('importPreviewUpdate')
71123
cy.get('.modal__content button').contains('Preview').click()
124+
cy.wait('@importPreviewUpdate')
72125
cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 3)
73126

74-
cy.intercept({ method: 'POST', url: '**/apps/tables/importupload/table/*/jobs' }).as('importUploadReq')
127+
cy.intercept({ method: 'POST', url: '**/apps/tables/v2/importupload/table/*' }).as('importUploadReq')
75128
cy.get('.modal__content button').contains('Import').click()
76-
cy.wait('@importUploadReq').its('response.statusCode').should('equal', 200)
129+
cy.wait('@importUploadReq')
130+
cy.get('[data-cy="importResultColumnsFound"]', { timeout: 20000 }).should('contain.text', '2')
131+
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '3')
132+
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
133+
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '0')
134+
cy.get('[data-cy="importResultRowsUpdated"]').should('contain.text', '1')
135+
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
136+
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
77137
})
78138

79139
})
80140

81141
describe('Import csv from Files file action', () => {
142+
const importToTablesAction = '[data-cy-files-list-row-action="import-to-tables"]'
143+
const openImportToTablesAction = (file, retries = 8) => {
144+
const rowActionsButton = `[data-cy-files-list-row-name="${file}"] [data-cy-files-list-row-actions] .action-item button`
145+
cy.get(rowActionsButton).click()
146+
cy.get('body').then($body => {
147+
if ($body.find(importToTablesAction).length > 0) {
148+
cy.get(importToTablesAction).first().click()
149+
return
150+
}
151+
152+
if (retries <= 0) {
153+
throw new Error(`Import to Tables action did not become available in file actions menu for file: ${file}`)
154+
}
155+
156+
cy.get('body').click(0, 0)
157+
cy.wait(1000)
158+
openImportToTablesAction(file, retries - 1)
159+
})
160+
}
82161

83-
before(function () {
162+
before(function() {
84163
cy.createRandomUser().then(user => {
85164
localUser = user
86165
cy.login(localUser)
87-
cy.uploadFile('test-import.csv', 'text/csv')
166+
cy.uploadFile('test-import-small.csv', 'text/csv')
167+
cy.uploadFile('test-import-large.csv', 'text/csv')
88168
})
89169
})
90170

91-
beforeEach(function () {
171+
beforeEach(function() {
92172
cy.login(localUser)
93-
cy.visit('apps/files/files')
173+
cy.visit('apps/files/files', { timeout: 120000 })
174+
cy.get('[data-cy-files-list-row-name="test-import-small.csv"]', { timeout: 120000 }).should('be.visible')
175+
cy.get('[data-cy-files-list-row-name="test-import-large.csv"]', { timeout: 120000 }).should('be.visible')
176+
})
177+
178+
it('Import small file to new table', () => {
179+
openImportToTablesAction('test-import-small.csv')
180+
181+
cy.intercept({
182+
method: 'POST',
183+
url: '**/apps/tables/import/table/*',
184+
}).as('importNewTableReq')
185+
cy.get('[data-cy="fileActionImportButton"]').click({ force: true })
186+
cy.wait('@importNewTableReq').its('response.statusCode').should('equal', 200)
187+
188+
cy.get('[data-cy="importResultColumnsFound"]').should('contain.text', '4')
189+
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '0')
190+
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '4')
191+
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
192+
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
193+
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
94194
})
95195

96-
it('Import to new table', () => {
97-
cy.get('[data-cy-files-list-row-name="test-import.csv"] [data-cy-files-list-row-actions] .action-item button').click()
98-
cy.get('[data-cy-files-list-row-action="import-to-tables"]').click()
196+
it('Import large file to new table', () => {
197+
openImportToTablesAction('test-import-large.csv')
99198

100199
cy.intercept({
101200
method: 'POST',
102-
url: '**/apps/tables/import/table/*/jobs',
201+
url: '**/apps/tables/v2/import/table/*',
103202
}).as('importNewTableReq')
104203
cy.get('[data-cy="fileActionImportButton"]').click({ force: true })
105204
cy.wait('@importNewTableReq').its('response.statusCode').should('equal', 200)
106205
})
107206

108-
it('Import to existing table', () => {
109-
cy.get('[data-cy-files-list-row-name="test-import.csv"] [data-cy-files-list-row-actions] .action-item button').click()
110-
cy.get('[data-cy-files-list-row-action="import-to-tables"]').click()
207+
it('Import small file to existing table', () => {
208+
openImportToTablesAction('test-import-small.csv')
111209

112210
cy.get('.modal__content [data-cy="importAsNewTableSwitch"] input').uncheck({ force: true })
113211
cy.get('[data-cy="selectExistingTableDropdown"]').type('Welcome to Nextcloud Tables!')
114212
cy.get('.name-parts').click()
115213

116214
cy.intercept({
117215
method: 'POST',
118-
url: '**/apps/tables/import/table/*/jobs',
216+
url: '**/apps/tables/v2/import/table/*',
119217
}).as('importExistingTableReq')
120218
cy.get('[data-cy="fileActionImportButton"]').click({ force: true })
121219
cy.wait('@importExistingTableReq').its('response.statusCode').should('equal', 200)
220+
221+
cy.get('[data-cy="importResultColumnsFound"]').should('contain.text', '4')
222+
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '4')
223+
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
224+
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
225+
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
226+
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
122227
})
123228

124229
})

‎cypress/e2e/tables-table.cy.js‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ describe('Manage a table', () => {
3737
})
3838

3939
it('Create with import', () => {
40-
cy.uploadFile('test-import.csv', 'text/csv')
40+
cy.uploadFile('test-import-small.csv', 'text/csv')
4141
cy.get('.icon-loading').should('not.exist')
4242
cy.get('[data-cy="navigationCreateTableIcon"]').click({ force: true })
4343
// should type before selecting the table type tile
@@ -47,13 +47,19 @@ describe('Manage a table', () => {
4747
cy.contains('h2', 'Import').should('be.visible')
4848

4949
cy.get('.modal__content button').contains('Select from Files').click()
50-
cy.get('.file-picker__files').contains('test-import').click()
50+
cy.get('.file-picker__files').contains('test-import-small').click()
5151
cy.get('.file-picker button span').contains('Import').click()
5252
cy.get('.modal__content button').contains('Preview').click()
5353
cy.get('.file_import__preview tbody tr').should('have.length', 4)
54-
cy.intercept({ method: 'POST', url: '**/apps/tables/import/table/*/jobs'}).as('importUploadReq')
54+
cy.intercept({ method: 'POST', url: '**/apps/tables/v2/import/table/*' }).as('importUploadReq')
5555
cy.get('.modal__content button').contains('Import').scrollIntoView().click()
56-
cy.wait('@importUploadReq').its('response.statusCode').should('equal', 200)
56+
cy.wait('@importUploadReq')
57+
cy.get('[data-cy="importResultColumnsFound"]').should('contain.text', '4')
58+
cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '0')
59+
cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '4')
60+
cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '3')
61+
cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
62+
cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
5763
})
5864

5965
it('Update description', () => {

0 commit comments

Comments
 (0)