smb: refuse OPEN_ANDX on directories

handleOpenAndX handed out a FID over os.OpenFile(dir) without checking
IsDir first, unlike OPEN/CREATE. The open itself succeeded (attrs
correctly reported Directory), so a directory-copy client only found
out on the follow-up READ, which failed with the generic
statusUnsuccessful (ERRSRV/ERRerror) — a code CORE-dialect clients
can't act on, so the whole copy aborted (seen client-side as Windows
"System error 1026" against an IPX SMB share).

Reject with STATUS_FILE_IS_A_DIRECTORY at open time instead, matching
handleOpen/handleCreate/handleNTCreateAndX.

Author: pgodwin

core/service/smb/fileio.go

@@ -63,6 +63,21 @@ func (s *Service) handleOpenAndX(sess *smbSession, h protocol.Header, req []byte
 	if st != statusSuccess {
 		return errResponse(h, st)
 	}
+	if info, err := sh.FS().Stat(store); err == nil && info.IsDir() {
+		// Unlike OPEN/CREATE, this path handed out a FID over
+		// os.OpenFile(dir) without checking IsDir first — the open succeeded
+		// (attrs correctly reported Directory), but the follow-up
+		// READ/READ_MPX on that FID then failed with the generic
+		// statusUnsuccessful (ERRSRV/ERRerror), a code CORE-dialect clients
+		// can't act on. A directory-copy walks FIND_FIRST2 results and opens
+		// each entry by name, depending on OPEN_ANDX itself rejecting
+		// directories to know to recurse instead of stream-reading it: an
+		// IPX SMB capture of a stalled directory copy showed the Open AndX
+		// Response reporting File Attributes Directory, then two Read
+		// attempts both coming back ERRSRV/ERRerror before the client's copy
+		// aborted (Windows-side "System error 1026").
+		return errResponse(h, statusFileIsADirectory)
+	}
 
 	// OpenFunction (low nibble = action if exists, high nibble = action if
 	// missing): 0x0001 open, 0x0002 truncate, 0x0010 create-if-missing. The

core/service/smb/fileio_test.go

@@ -313,3 +313,31 @@ func TestFS_OpenMissingFileNotFound(t *testing.T) {
 		t.Fatalf("OPEN_ANDX missing status = %#x, want OBJECT_NAME_NOT_FOUND", h.Status)
 	}
 }
+
+// TestFS_OpenAndXOnDirectoryRefused proves OPEN_ANDX (0x2D) on a directory is
+// refused with STATUS_FILE_IS_A_DIRECTORY, matching OPEN/CREATE. Before this,
+// OPEN_ANDX had no IsDir check: it handed out a FID over os.OpenFile(dir), which
+// succeeded (attrs correctly reported Directory), and only the follow-up READ
+// failed — with the generic ERRSRV/ERRerror a CORE-dialect directory-copy client
+// can't distinguish from an ordinary I/O fault, so it aborted the whole copy
+// instead of recursing (observed as Windows "System error 1026" over an IPX SMB
+// capture).
+func TestFS_OpenAndXOnDirectoryRefused(t *testing.T) {
+	svc, sess, tid := fsService(t)
+
+	mkdirReq := smbReq(protocol.CommandCreateDirectory, protocol.Flags2NTStatus, tid, 1, nil, ansiPathArea("subdir"))
+	if h := respHeader(t, svc.Dispatch(sess, mkdirReq)); h.Status != statusSuccess {
+		t.Fatalf("CREATE_DIRECTORY status = %#x", h.Status)
+	}
+
+	ow := make([]byte, 30)
+	ow[0] = protocol.CommandNoAndXCommand
+	bp.PutLE16(ow[16:18], 0x01) // open existing, no create bit
+	reply := svc.Dispatch(sess, smbReq(protocol.CommandOpenAndX, protocol.Flags2NTStatus, tid, 1, ow, ansiPathArea("subdir")))
+	if h := respHeader(t, reply); h.Status != statusFileIsADirectory {
+		t.Fatalf("OPEN_ANDX on directory status = %#x, want STATUS_FILE_IS_A_DIRECTORY", h.Status)
+	}
+	if len(sess.fids) != 0 {
+		t.Fatalf("OPEN_ANDX on directory left %d FID(s) open, want 0", len(sess.fids))
+	}
+}