Enhancement: Update rows via import

Signed-off-by: Kostiantyn Miakshyn <molodchick@gmail.com>

Author: Koc

cypress/e2e/tables-import.cy.js

@@ -61,6 +61,40 @@ describe('Import csv', () => {
 		cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
 	})
 
+	it('Import csv from device with updating of existent files', () => {
+		cy.intercept({ method: 'GET', url: '**/apps/tables/row/table/*' }).as('rowsReq')
+
+		cy.loadTable('Welcome to Nextcloud Tables!')
+
+		cy.wait('@rowsReq').then(({ response }) => {
+			const firstRow = response.body[0]
+			const csv = [
+				['id', 'What', 'How to do'],
+				[firstRow.id, 'What (Updated)', 'How to do (Updated)'],
+			]
+
+			cy.writeFile('cypress/fixtures/test-import-update.csv', csv.map(row => row.join(',')).join('\n'))
+		})
+
+		cy.clickOnTableThreeDotMenu('Import')
+		cy.get('.modal__content button').contains('Upload from device').click()
+		cy.get('input[type="file"]').selectFile('cypress/fixtures/test-import-update.csv', { force: true })
+
+		cy.get('.modal__content button').contains('Preview').click()
+		cy.get('.file_import__preview tbody tr', { timeout: 20000 }).should('have.length', 3)
+
+		cy.intercept({ method: 'POST', url: '**/apps/tables/importupload/table/*'}).as('importUploadReq')
+		cy.get('.modal__content button').contains('Import').click()
+		cy.wait('@importUploadReq')
+		cy.get('[data-cy="importResultColumnsFound"]', { timeout: 20000 }).should('contain.text', '2')
+		cy.get('[data-cy="importResultColumnsMatch"]').should('contain.text', '3')
+		cy.get('[data-cy="importResultColumnsCreated"]').should('contain.text', '0')
+		cy.get('[data-cy="importResultRowsInserted"]').should('contain.text', '0')
+		cy.get('[data-cy="importResultRowsUpdated"]').should('contain.text', '1')
+		cy.get('[data-cy="importResultParsingErrors"]').should('contain.text', '0')
+		cy.get('[data-cy="importResultRowErrors"]').should('contain.text', '0')
+	})
+
 })
 
 describe('Import csv from Files file action', () => {

lib/Controller/Api1Controller.php

@@ -1348,7 +1348,7 @@ public function updateRow(int $rowId, ?int $viewId, $data): DataResponse {
 		}
 
 		try {
-			return new DataResponse($this->rowService->updateSet($rowId, $viewId, $dataNew, $this->userId)->jsonSerialize());
+			return new DataResponse($this->rowService->updateSet($rowId, $viewId, $dataNew, $this->userId, null)->jsonSerialize());
 		} catch (BadRequestError $e) {
 			$this->logger->warning('An bad request was encountered: ' . $e->getMessage(), ['exception' => $e]);
 			return new DataResponse(['message' => $e->translatedMessage ?: $e->getMessage()], Http::STATUS_BAD_REQUEST);

lib/Controller/RowController.php

@@ -64,7 +64,7 @@ public function update(
 			$columnId,
 			$data
 		) {
-			return $this->service->updateSet($id, $viewId, ['columnId' => $columnId, 'value' => $data], $this->userId);
+			return $this->service->updateSet($id, $viewId, ['columnId' => $columnId, 'value' => $data], $this->userId, null);
 		});
 	}
 
@@ -73,18 +73,13 @@ public function updateSet(
 		int $id,
 		?int $viewId,
 		array $data,
-
 	): DataResponse {
 		return $this->handleError(function () use (
 			$id,
 			$viewId,
 			$data
 		) {
-			return $this->service->updateSet(
-				$id,
-				$viewId,
-				$data,
-				$this->userId);
+			return $this->service->updateSet($id, $viewId, $data, $this->userId, null);
 		});
 	}
 

lib/Db/Column.php

@@ -108,6 +108,8 @@ class Column extends EntitySuper implements JsonSerializable {
 
 	public const SUBTYPE_TEXT_LINE = 'line';
 
+	public const META_ID_TITLE = 'id';
+
 	protected ?string $title = null;
 	protected ?int $tableId = null;
 	protected ?string $createdBy = null;

lib/ResponseDefinitions.php

@@ -147,6 +147,7 @@
  *  matching_columns_count: int,
  *  created_columns_count: int,
  *  inserted_rows_count: int,
+ *  updated_rows_count: int,
  *  errors_parsing_count: int,
  *  errors_count: int,
  * }

lib/Service/ColumnService.php

@@ -473,7 +473,7 @@ public function findOrCreateColumnsByTitleForTableAsArray(?int $tableId, ?int $v
 				$result[$i] = '';
 			}
 			// if column was not found
-			if ($result[$i] === '' && $createUnknownColumns) {
+			if ($result[$i] === '' && $createUnknownColumns && $dataTypes[$i]['type'] !== Column::TYPE_META_ID) {
 				$description = $this->l->t('This column was automatically created by the import service.');
 				$result[$i] = $this->create(
 					$userId,

lib/Service/ImportService.php

@@ -34,7 +34,6 @@
 use Psr\Container\ContainerExceptionInterface;
 use Psr\Container\NotFoundExceptionInterface;
 use Psr\Log\LoggerInterface;
-use Throwable;
 use TypeError;
 use function file_exists;
 use function is_string;
@@ -55,9 +54,11 @@ class ImportService extends SuperService {
 	private ?int $viewId = null;
 	private array $columns = [];
 	private bool $createUnknownColumns = true;
+	private ?int $idColumnIndex = null;
 	private int $countMatchingColumns = 0;
 	private int $countCreatedColumns = 0;
 	private int $countInsertedRows = 0;
+	private int $countUpdatedRows = 0;
 	private int $countErrors = 0;
 	private int $countParsingErrors = 0;
 
@@ -152,14 +153,19 @@ private function getPreviewData(Worksheet $worksheet): array {
 				$column = $this->columns[$colIndex];
 				$columns[] = $column;
 			} else {
-				$columns[] = [
+				$column = [
 					'title' => $title,
 					'type' => $this->rawColumnDataTypes[$colIndex]['type'],
 					'subtype' => $this->rawColumnDataTypes[$colIndex]['subtype'] ?? null,
 					'numberDecimals' => $this->rawColumnDataTypes[$colIndex]['number_decimals'] ?? 0,
 					'numberPrefix' => $this->rawColumnDataTypes[$colIndex]['number_prefix'] ?? '',
 					'numberSuffix' => $this->rawColumnDataTypes[$colIndex]['number_suffix'] ?? '',
 				];
+				if (mb_strtolower($title) === Column::META_ID_TITLE) {
+					$column['id'] = Column::TYPE_META_ID;
+				}
+
+				$columns[] = $column;
 			}
 		}
 
@@ -179,7 +185,7 @@ private function getPreviewData(Worksheet $worksheet): array {
 				$colIndex = $cellIterator->getCurrentColumnIndex() - 1;
 				$column = $this->columns[$colIndex];
 
-				if (!array_key_exists($colIndex, $this->columns)) {
+				if (!array_key_exists($colIndex, $columns)) {
 					continue;
 				}
 
@@ -324,6 +330,7 @@ public function import(?int $tableId, ?int $viewId, string $path, bool $createMi
 			'matching_columns_count' => $this->countMatchingColumns,
 			'created_columns_count' => $this->countCreatedColumns,
 			'inserted_rows_count' => $this->countInsertedRows,
+			'updated_rows_count' => $this->countUpdatedRows,
 			'errors_parsing_count' => $this->countParsingErrors,
 			'errors_count' => $this->countErrors,
 		];
@@ -354,7 +361,7 @@ private function loop(Worksheet $worksheet): void {
 
 		foreach ($worksheet->getRowIterator(2) as $row) {
 			// parse row data
-			$this->createRow($row);
+			$this->upsertRow($row);
 		}
 	}
 
@@ -387,25 +394,46 @@ private function parseValueByColumnType(string $value, Column $column): string {
 	 * @throws MultipleObjectsReturnedException
 	 * @throws NotFoundError
 	 */
-	private function createRow(Row $row): void {
+	private function upsertRow(Row $row): void {
 		$cellIterator = $row->getCellIterator();
 		$cellIterator->setIterateOnlyExistingCells(false);
 
 		try {
 			$i = -1;
 			$data = [];
 			$hasData = false;
+			$id = null;
 			foreach ($cellIterator as $cell) {
 				$i++;
 
+				if ($this->idColumnIndex !== null && $i === $this->idColumnIndex) {
+					// if this is the ID column, we need to get the ID from the cell
+					if ($cell && $cell->getValue() !== null) {
+						$id = $cell->getValue();
+					}
+					if ($id !== null && !is_numeric($id)) {
+						$this->logger->warning('ID column value is not numeric: ' . $id);
+						$this->countErrors++;
+						return;
+					}
+					$id = (int)$id;
+					continue;
+				}
+
+				$columnKey = $i;
+				if ($this->columnsConfig && $this->idColumnIndex !== null && $i > $this->idColumnIndex) {
+					// if we have an ID column, we need to adjust the index
+					$columnKey = $i - 1;
+				}
+
 				// only add the dataset if column is known
-				if (!isset($this->columns[$i]) || $this->columns[$i] === '') {
+				if (!isset($this->columns[$columnKey]) || $this->columns[$columnKey] === '') {
 					$this->logger->debug('Column unknown while fetching rows data for importing.');
 					continue;
 				}
 
 				/** @var Column $column */
-				$column = $this->columns[$i];
+				$column = $this->columns[$columnKey];
 
 				// if cell is empty
 				if (!$cell || $cell->getValue() === null) {
@@ -446,26 +474,31 @@ private function createRow(Row $row): void {
 				];
 			}
 
-			if ($hasData) {
+			if (!$hasData) {
+				$this->logger->debug('Skipped empty row ' . $row->getRowIndex() . ' during import');
+				return;
+			}
+
+			if ($id) {
+				$this->rowService->updateSet($id, $this->viewId, $data, $this->userId, $this->tableId);
+				$this->countUpdatedRows++;
+			} else {
 				$this->rowService->create($this->tableId, $this->viewId, $data);
 				$this->countInsertedRows++;
-			} else {
-				$this->logger->debug('Skipped empty row ' . $row->getRowIndex() . ' during import');
 			}
 		} catch (PermissionError $e) {
-			$this->logger->error('Could not create row while importing, no permission.', ['exception' => $e]);
+			$this->logger->error('Could not create/update row while importing, no permission.', ['exception' => $e]);
 			$this->countErrors++;
 		} catch (BadRequestError|InternalError $e) {
-			$this->logger->error('Error while creating  new row for import.', ['exception' => $e]);
+			$this->logger->error('Error while creating/updating  new row for import.', ['exception' => $e]);
 			$this->countErrors++;
 		} catch (NotFoundError $e) {
 			$this->logger->error($e->getMessage(), ['exception' => $e]);
-			throw new NotFoundError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage());
-		} catch (Throwable $e) {
+			throw new NotFoundError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), 0, $e);
+		} catch (\Throwable $e) {
 			$this->countErrors++;
-			$this->logger->error('Error while creating new row for import.', ['exception' => $e]);
+			$this->logger->error('Error while creating/updating new row for import.', ['exception' => $e]);
 		}
-
 	}
 
 	private function valueToDateTimeImmutable(mixed $value): ?DateTimeImmutable {
@@ -522,9 +555,25 @@ private function getColumns(Row $firstRow, Row $secondRow): void {
 			if ($cell && $cell->getValue() !== null && $cell->getValue() !== '') {
 				$title = $cell->getValue();
 
+				if (!$this->columnsConfig && mb_strtolower($title) === Column::META_ID_TITLE) {
+					$this->idColumnIndex = $index;
+					$titles[] = $title;
+					$dataTypes[] = $this->parseColumnDataType($secondRowCellIterator->current());
+					$secondRowCellIterator->next();
+					$index++;
+					continue;
+				}
 				if (isset($this->columnsConfig[$index]) && $this->columnsConfig[$index]['action'] === 'exist' && $this->columnsConfig[$index]['existColumn']) {
 					$title = $this->columnsConfig[$index]['existColumn']['label'];
 					$countMatchingColumnsFromConfig++;
+
+					// no need to create the ID (Meta) column as it used for update
+					if ($this->columnsConfig[$index]['existColumn']['id'] === Column::TYPE_META_ID) {
+						$this->idColumnIndex = $index;
+						$secondRowCellIterator->next();
+						$index++;
+						continue;
+					}
 				}
 				if (isset($this->columnsConfig[$index]) && $this->columnsConfig[$index]['action'] === 'new' && $this->createUnknownColumns) {
 					$column = $this->columnService->create(

lib/Service/RowService.php

@@ -410,6 +410,7 @@ public function updateSet(
 		?int $viewId,
 		array $data,
 		string $userId,
+		?int $tableId,
 	): Row2 {
 		try {
 			$item = $this->getRowById($id);
@@ -459,8 +460,14 @@ public function updateSet(
 				throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage());
 			}
 		} else {
-			// if no view id is set, we assume a table and take the tableId from the row
-			$tableId = $item->getTableId();
+			if ($tableId === null) {
+				$tableId = $item->getTableId();
+			}
+			if ($tableId !== $item->getTableId()) {
+				$e = new \Exception('Row does not belong to table with id ' . $tableId);
+				$this->logger->error($e->getMessage(), ['exception' => $e]);
+				throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage());
+			}
 
 			// security
 			if (!$this->permissionsService->canReadRowsByElementId($item->getTableId(), 'table', $userId)) {

openapi.json

@@ -310,6 +310,7 @@
                     "matching_columns_count",
                     "created_columns_count",
                     "inserted_rows_count",
+                    "updated_rows_count",
                     "errors_parsing_count",
                     "errors_count"
                 ],
@@ -330,6 +331,10 @@
                         "type": "integer",
                         "format": "int64"
                     },
+                    "updated_rows_count": {
+                        "type": "integer",
+                        "format": "int64"
+                    },
                     "errors_parsing_count": {
                         "type": "integer",
                         "format": "int64"

src/modules/modals/ImportPreview.vue

@@ -70,6 +70,7 @@ import { ColumnTypes } from '../../shared/components/ncTable/mixins/columnHandle
 import { emit } from '@nextcloud/event-bus'
 import { useTablesStore } from '../../store/store.js'
 import { useDataStore } from '../../store/data.js'
+import { TYPE_META_ID } from '../../shared/constants.js'
 
 export default {
 	name: 'ImportPreview',
@@ -116,10 +117,17 @@ export default {
 				return []
 			}
 
-			return this.existingColumns.map(column => ({
+			const columns = this.existingColumns.map(column => ({
 				id: column.id,
 				label: column.title,
 			}))
+
+			columns.unshift({
+				id: TYPE_META_ID,
+				label: t('tables', 'ID (Meta)'),
+			})
+
+			return columns
 		},
 	},