Enhancement: Update rows via import

Signed-off-by: Kostiantyn Miakshyn <molodchick@gmail.com>

Author: Koc

lib/Db/Column.php

@@ -101,6 +101,8 @@ class Column extends EntitySuper implements JsonSerializable {
 
 	public const SUBTYPE_TEXT_LINE = 'line';
 
+	public const META_ID_TITLE = 'id';
+
 	protected ?string $title = null;
 	protected ?int $tableId = null;
 	protected ?string $createdBy = null;

lib/ResponseDefinitions.php

@@ -138,6 +138,7 @@
  *  matching_columns_count: int,
  *  created_columns_count: int,
  *  inserted_rows_count: int,
+ *  updated_rows_count: int,
  *  errors_parsing_count: int,
  *  errors_count: int,
  * }

lib/Service/ImportService.php

@@ -48,9 +48,11 @@ class ImportService extends SuperService {
 	private ?int $viewId = null;
 	private array $columns = [];
 	private bool $createUnknownColumns = true;
+	private ?int $idColumnIndex = null;
 	private int $countMatchingColumns = 0;
 	private int $countCreatedColumns = 0;
 	private int $countInsertedRows = 0;
+	private int $countUpdatedRows = 0;
 	private int $countErrors = 0;
 	private int $countParsingErrors = 0;
 
@@ -136,14 +138,19 @@ private function getPreviewData(Worksheet $worksheet): array {
 				$column = $this->columns[$colIndex];
 				$columns[] = $column;
 			} else {
-				$columns[] = [
+				$column = [
 					'title' => $title,
 					'type' => $this->rawColumnDataTypes[$colIndex]['type'],
 					'subtype' => $this->rawColumnDataTypes[$colIndex]['subtype'] ?? null,
 					'numberDecimals' => $this->rawColumnDataTypes[$colIndex]['number_decimals'] ?? 0,
 					'numberPrefix' => $this->rawColumnDataTypes[$colIndex]['number_prefix'] ?? '',
 					'numberSuffix' => $this->rawColumnDataTypes[$colIndex]['number_suffix'] ?? '',
 				];
+				if (mb_strtolower($title) === Column::META_ID_TITLE) {
+					$column['id'] = Column::TYPE_META_ID;
+				}
+
+				$columns[] = $column;
 			}
 		}
 
@@ -304,6 +311,7 @@ public function import(?int $tableId, ?int $viewId, string $path, bool $createMi
 			'matching_columns_count' => $this->countMatchingColumns,
 			'created_columns_count' => $this->countCreatedColumns,
 			'inserted_rows_count' => $this->countInsertedRows,
+			'updated_rows_count' => $this->countUpdatedRows,
 			'errors_parsing_count' => $this->countParsingErrors,
 			'errors_count' => $this->countErrors,
 		];
@@ -334,7 +342,7 @@ private function loop(Worksheet $worksheet): void {
 
 		foreach ($worksheet->getRowIterator(2) as $row) {
 			// parse row data
-			$this->createRow($row);
+			$this->upsertRow($row);
 		}
 	}
 
@@ -367,25 +375,46 @@ private function parseValueByColumnType(string $value, Column $column): string {
 	 * @throws MultipleObjectsReturnedException
 	 * @throws NotFoundError
 	 */
-	private function createRow(Row $row): void {
+	private function upsertRow(Row $row): void {
 		$cellIterator = $row->getCellIterator();
 		$cellIterator->setIterateOnlyExistingCells(false);
 
 		try {
 			$i = -1;
 			$data = [];
 			$hasData = false;
+			$id = null;
 			foreach ($cellIterator as $cell) {
 				$i++;
 
+				if ($this->idColumnIndex !== null && $i === $this->idColumnIndex) {
+					// if this is the ID column, we need to get the ID from the cell
+					if ($cell && $cell->getValue() !== null) {
+						$id = $cell->getValue();
+					}
+					if ($id !== null && !is_numeric($id)) {
+						$this->logger->warning('ID column value is not numeric: ' . $id);
+						$this->countErrors++;
+						return;
+					}
+					$id = (int)$id;
+					continue;
+				}
+
+				$columnKey = $i;
+				if ($this->idColumnIndex !== null && $i > $this->idColumnIndex) {
+					// if we have an ID column, we need to adjust the index
+					$columnKey = $i - 1;
+				}
+
 				// only add the dataset if column is known
-				if (!isset($this->columns[$i]) || $this->columns[$i] === '') {
+				if (!isset($this->columns[$columnKey]) || $this->columns[$columnKey] === '') {
 					$this->logger->debug('Column unknown while fetching rows data for importing.');
 					continue;
 				}
 
 				/** @var Column $column */
-				$column = $this->columns[$i];
+				$column = $this->columns[$columnKey];
 
 				// if cell is empty
 				if (!$cell || $cell->getValue() === null) {
@@ -426,26 +455,36 @@ private function createRow(Row $row): void {
 				];
 			}
 
-			if ($hasData) {
-				$this->rowService->create($this->tableId, $this->viewId, $data);
-				$this->countInsertedRows++;
-			} else {
+			if (!$hasData) {
 				$this->logger->debug('Skipped empty row ' . $row->getRowIndex() . ' during import');
+				return;
+			}
+
+			if ($id) {
+				try {
+					$this->rowService->updateSet($id, $this->viewId, $data, $this->userId);
+					$this->countUpdatedRows++;
+					return;
+				} catch (\Throwable $e) {
+					$this->logger->warning('Error while updating row for import.', ['row_id' => $id, 'exception' => $e]);
+				}
 			}
+
+			$this->rowService->create($this->tableId, $this->viewId, $data);
+			$this->countInsertedRows++;
 		} catch (PermissionError $e) {
-			$this->logger->error('Could not create row while importing, no permission.', ['exception' => $e]);
+			$this->logger->error('Could not create/update row while importing, no permission.', ['exception' => $e]);
 			$this->countErrors++;
 		} catch (InternalError $e) {
-			$this->logger->error('Error while creating  new row for import.', ['exception' => $e]);
+			$this->logger->error('Error while creating/updating new row for import.', ['exception' => $e]);
 			$this->countErrors++;
 		} catch (NotFoundError $e) {
 			$this->logger->error($e->getMessage(), ['exception' => $e]);
-			throw new NotFoundError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage());
+			throw new NotFoundError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), 0, $e);
 		} catch (\Throwable $e) {
 			$this->countErrors++;
-			$this->logger->error('Error while creating new row for import.', ['exception' => $e]);
+			$this->logger->error('Error while creating/updating new row for import.', ['exception' => $e]);
 		}
-
 	}
 
 	private function valueToDateTimeImmutable(mixed $value): ?\DateTimeImmutable {
@@ -502,9 +541,23 @@ private function getColumns(Row $firstRow, Row $secondRow): void {
 			if ($cell && $cell->getValue() !== null && $cell->getValue() !== '') {
 				$title = $cell->getValue();
 
+				if (!$this->columnsConfig && mb_strtolower($title) === Column::META_ID_TITLE) {
+					$this->idColumnIndex = $index;
+					$secondRowCellIterator->next();
+					$index++;
+					continue;
+				}
 				if (isset($this->columnsConfig[$index]) && $this->columnsConfig[$index]['action'] === 'exist' && $this->columnsConfig[$index]['existColumn']) {
 					$title = $this->columnsConfig[$index]['existColumn']['label'];
 					$countMatchingColumnsFromConfig++;
+
+					// no need to create the ID (Meta) column as it used for update
+					if ($this->columnsConfig[$index]['existColumn']['id'] === Column::TYPE_META_ID) {
+						$this->idColumnIndex = $index;
+						$secondRowCellIterator->next();
+						$index++;
+						continue;
+					}
 				}
 				if (isset($this->columnsConfig[$index]) && $this->columnsConfig[$index]['action'] === 'new' && $this->createUnknownColumns) {
 					$column = $this->columnService->create(

openapi.json

@@ -269,6 +269,7 @@
                     "matching_columns_count",
                     "created_columns_count",
                     "inserted_rows_count",
+                    "updated_rows_count",
                     "errors_parsing_count",
                     "errors_count"
                 ],
@@ -289,6 +290,10 @@
                         "type": "integer",
                         "format": "int64"
                     },
+                    "updated_rows_count": {
+                        "type": "integer",
+                        "format": "int64"
+                    },
                     "errors_parsing_count": {
                         "type": "integer",
                         "format": "int64"

src/modules/modals/ImportPreview.vue

@@ -70,6 +70,7 @@ import { ColumnTypes } from '../../shared/components/ncTable/mixins/columnHandle
 import { emit } from '@nextcloud/event-bus'
 import { useTablesStore } from '../../store/store.js'
 import { useDataStore } from '../../store/data.js'
+import { TYPE_META_ID } from '../../shared/constants.js'
 
 export default {
 	name: 'ImportPreview',
@@ -116,10 +117,17 @@ export default {
 				return []
 			}
 
-			return this.existingColumns.map(column => ({
+			const columns = this.existingColumns.map(column => ({
 				id: column.id,
 				label: column.title,
 			}))
+
+			columns.unshift({
+				id: TYPE_META_ID,
+				label: t('tables', 'ID (Meta)'),
+			})
+
+			return columns
 		},
 	},
 

src/modules/modals/ImportResults.vue

@@ -37,6 +37,13 @@
 				</td>
 			</tr>
 
+			<tr>
+				<td>{{ t('tables', 'Updated rows') }}</td>
+				<td data-cy="importResultRowsUpdated">
+					{{ results.updated_rows_count }}
+				</td>
+			</tr>
+
 			<tr>
 				<td>{{ t('tables', 'Value parsing errors') }}</td>
 				<td data-cy="importResultParsingErrors">

src/shared/components/ncTable/mixins/exportTableMixin.js

@@ -21,7 +21,7 @@ export default {
 
 			const data = []
 			rows.forEach(row => {
-				const rowData = {}
+				const rowData = { ID: row.id }
 				columns.forEach(column => {
 					// if a normal column
 					if (column.id >= 0) {

src/types/openapi/openapi.ts

@@ -814,6 +814,8 @@ export type components = {
             /** Format: int64 */
             readonly inserted_rows_count: number;
             /** Format: int64 */
+            readonly updated_rows_count: number;
+            /** Format: int64 */
             readonly errors_parsing_count: number;
             /** Format: int64 */
             readonly errors_count: number;

tests/integration/features/APIv1.feature

@@ -248,6 +248,38 @@ Feature: APIv1
     | import-from-ms365.xlsx       |
     | import-from-libreoffice.csv  |
 
+  @api1 @import @rows
+  Scenario: Import a document file that updates existing rows
+    Given table "Import check" with emoji "👨🏻‍💻" exists for user "participant1" as "base1"
+    Then column "one" exists with following properties
+      | type        | text                   |
+      | subtype     | line                   |
+      | mandatory   | 1                      |
+      | description | This is a description! |
+    Then column "two" exists with following properties
+      | type        | number                 |
+      | mandatory   | 1                      |
+      | description | This is a description! |
+    Then row exists with following values
+      | one | AHA |
+      | two | 88  |
+    Given file "update-rows.csv" exists for user "participant1" with the following data
+      | ID      | one         | two |
+      | {rowId} | AHA updated | 99  |
+      |         | new row     | 100 |
+    When user imports file "update-rows.csv" into last created table
+    Then import results have the following data
+      | found_columns_count    | 2 |
+      | matching_columns_count | 2 |
+      | created_columns_count  | 0 |
+      | inserted_rows_count    | 1 |
+      | updated_rows_count     | 1 |
+      | errors_count           | 0 |
+    Then table contains at least following rows
+      | one         | two |
+      | AHA updated | 99  |
+      | new row     | 100 |
+
   @api1 @import
   Scenario: Import a document with optional field
     Given user "participant1" uploads file "import-from-libreoffice-optional-fields.csv"

tests/integration/features/bootstrap/FeatureContext.php

@@ -489,35 +489,40 @@ public function uploadFile(string $user, string $file): void {
 	// IMPORT --------------------------
 
 	/**
-	 * @Given file :file exists for user :user with following data
+	 * @Given file :file exists for user :user with the following data
 	 *
 	 * @param string $user
 	 * @param string $file
-	 * @param TableNode|null $table
+	 * @param TableNode $table
 	 */
-	public function createCsvFile(string $user, string $file, ?TableNode $table = null): void {
+	public function createCsvFile(string $user, string $file, TableNode $table): void {
 		$this->setCurrentUser($user);
-		$url = $this->baseUrl . 'remote.php/dav/files/' . $user . $file;
-		$body = $this->tableNodeToCsv($table);
-		$headers = ['Content-Type' => 'text/csv'];
 
-		$this->sendRequestFullUrl('PUT', $url, $body, $headers, []);
+		$url = sprintf('%sremote.php/dav/files/%s/%s', $this->baseUrl, $user, $file);
+		$body = Utils::streamFor($this->tableNodeToCsv($table));
+
+		$this->sendRequestFullUrl('PUT', $url, $body);
 
 		Assert::assertEquals(201, $this->response->getStatusCode());
 	}
 
 	private function tableNodeToCsv(TableNode $node): string {
-		$out = '';
+		$file = new \SplTempFileObject();
 		foreach ($node->getRows() as $row) {
-			foreach ($row as $value) {
-				if ($out !== '' && substr($out, -1) !== "\n") {
-					$out .= ',';
-				}
-				$out .= trim($value);
-			}
-			$out .= "\n";
+			$fields = array_map(function ($cell) {
+				return str_replace('{rowId}', $this->rowId, $cell);
+			}, $row);
+			$file->fputcsv($fields);
 		}
-		return $out;
+
+		$file->rewind();
+
+		$lines = '';
+		while (!$file->eof()) {
+			$lines .= $file->fgets();
+		}
+
+		return $lines;
 	}
 
 	/**