fix(filesystem): do not attempt to lock directories on Windows

LockFile() locks a byte range within a file and is not supported for
directory handles, where it always fails with ERROR_INVALID_PARAMETER.
FILE_FLAG_BACKUP_SEMANTICS lets CreateFileW() open directories, so
isFileLocked() reached that failing call for every directory and logged a
warning for each one. Discovery checks every entry, directories included,
so a sync run logged one bogus warning per directory. On large folder trees
those warnings dominate the log volume, and the resulting log rotation
discards the records needed to diagnose actual problems.

Return the opened handle for directories instead of attempting the lock.
CreateFileW() still runs, so a directory held with deny-sharing is still
reported as locked; only the attempt that cannot succeed is gone.

Measured on a synced folder with 82464 directories, same build and
configuration: a full discovery run logged 83189 of these warnings without
this change and none with it. The added test covers both halves and fails
without the fix. Test suite green on Windows (70 tests).

For #10444

Assisted-by: Claude Code:claude-opus-5
Signed-off-by: Andreas Bohl <ab@eeloy.com>

Author: andreasbohl

src/common/filesystembase.cpp

@@ -702,6 +702,12 @@ Utility::Handle lockFile(const QString &fileName, FileSystem::LockMode mode)
                                                FILE_ATTRIBUTE_NORMAL | FILE_FLAG_BACKUP_SEMANTICS, nullptr)};
 
         if (out) {
+            if (attr & FILE_ATTRIBUTE_DIRECTORY) {
+                // LockFile() is unsupported for directory handles and always fails there,
+                // and opening the directory already ruled out a sharing violation.
+                return out;
+            }
+
             LARGE_INTEGER start;
             start.QuadPart = 0;
             LARGE_INTEGER end;

test/testlockedfiles.cpp

@@ -9,6 +9,7 @@
  */
 
 #include <QtTest>
+#include <QDir>
 #include "syncenginetestutils.h"
 #include "lockwatcher.h"
 #include <syncengine.h>
@@ -34,6 +35,23 @@ HANDLE makeHandle(const QString &file, int shareMode)
     }
     return handle;
 }
+
+// Same as makeHandle(), but FILE_FLAG_BACKUP_SEMANTICS is required to open a directory.
+HANDLE makeDirectoryHandle(const QString &directory, int shareMode)
+{
+    const auto fName = FileSystem::longWinPath(directory);
+    auto handle = CreateFileW(
+        reinterpret_cast<const wchar_t *>(fName.utf16()),
+        GENERIC_READ,
+        shareMode,
+        nullptr, OPEN_EXISTING,
+        FILE_ATTRIBUTE_NORMAL | FILE_FLAG_BACKUP_SEMANTICS,
+        nullptr);
+    if (handle == INVALID_HANDLE_VALUE) {
+        qWarning() << GetLastError();
+    }
+    return handle;
+}
 #endif
 
 class TestLockedFiles : public QObject
@@ -112,6 +130,74 @@ private slots:
     }
 
 #ifdef Q_OS_WIN
+    void testDirectoryLockChecks()
+    {
+        QTemporaryDir tmp;
+        QVERIFY(tmp.isValid());
+
+        const auto subDirectory = tmp.path() + QStringLiteral("/aDirectory");
+        QVERIFY(QDir().mkpath(subDirectory));
+
+        const auto fileInDirectory = subDirectory + QStringLiteral("/aFile.txt");
+        {
+            QFile file(fileInDirectory);
+            QVERIFY(file.open(QFile::WriteOnly));
+            QVERIFY(file.write("Nextcloud"));
+        }
+
+        const auto fileHandle = makeHandle(fileInDirectory, 0);
+        QVERIFY(fileHandle != INVALID_HANDLE_VALUE);
+
+        // Logger only forwards fatal messages, so QTest::failOnWarning() would not see
+        // these. Count by category, which survives rewording of the message.
+        static int warningCount = 0;
+        static QStringList warningMessages;
+        warningCount = 0;
+        warningMessages.clear();
+        const auto previousHandler = qInstallMessageHandler([](QtMsgType type, const QMessageLogContext &context, const QString &message) {
+            if (type == QtWarningMsg && context.category && qstrcmp(context.category, "nextcloud.sync.filesystem") == 0) {
+                ++warningCount;
+                warningMessages.append(message);
+            }
+        });
+
+        // Every mode has to stay silent, but only SharedRead is asserted on: Exclusive
+        // requests deny-sharing, so any unrelated handle an indexer or virus scanner
+        // holds would rightfully make it report the directory as locked.
+        QVarLengthArray<bool, 2> sharedReadResults;
+        for (const auto mode : {FileSystem::LockMode::Shared, FileSystem::LockMode::SharedRead, FileSystem::LockMode::Exclusive}) {
+            const auto subDirectoryLocked = FileSystem::isFileLocked(subDirectory, mode);
+            const auto rootDirectoryLocked = FileSystem::isFileLocked(tmp.path(), mode);
+            if (mode == FileSystem::LockMode::SharedRead) {
+                sharedReadResults.append(subDirectoryLocked);
+                sharedReadResults.append(rootDirectoryLocked);
+            }
+        }
+
+        // Skipping the lock on directories must not hide a locked file inside one.
+        const auto lockedFileDetected = FileSystem::isFileLocked(fileInDirectory, FileSystem::LockMode::SharedRead);
+
+        // A directory held with deny-sharing is still locked; CreateFileW reports that.
+        const auto directoryHandle = makeDirectoryHandle(subDirectory, 0);
+        const auto sharedDirectoryDetected = directoryHandle != INVALID_HANDLE_VALUE
+            && FileSystem::isFileLocked(subDirectory, FileSystem::LockMode::SharedRead);
+
+        qInstallMessageHandler(previousHandler);
+        if (directoryHandle != INVALID_HANDLE_VALUE) {
+            CloseHandle(directoryHandle);
+        }
+        CloseHandle(fileHandle);
+
+        // The failing LockFile() call used to log one warning per directory per run.
+        QVERIFY2(warningCount == 0, qPrintable(warningMessages.join(QStringLiteral(" || "))));
+        for (const auto isLocked : sharedReadResults) {
+            QVERIFY(!isLocked);
+        }
+        QVERIFY(lockedFileDetected);
+        QVERIFY(sharedDirectoryDetected);
+        QVERIFY(!FileSystem::isFileLocked(fileInDirectory, FileSystem::LockMode::SharedRead));
+    }
+
     void testLockedFilePropagation()
     {
         FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };