Float fewer xwayland windows v2

include/wm/wm_server.h

@@ -1,6 +1,7 @@
 #ifndef WM_SERVER_H
 #define WM_SERVER_H
 
+#include "xwayland/xwm.h"
 #include <time.h>
 #include <wayland-server.h>
 #include <wlr/backend.h>
@@ -12,13 +13,30 @@
 #include <wlr/types/wlr_virtual_pointer_v1.h>
 #include <wlr/types/wlr_layer_shell_v1.h>
 
+#ifdef WM_HAS_XWAYLAND
+#include <xcb/xproto.h>
+#endif
+
 struct wm_config;
 struct wm_seat;
 struct wm_layout;
 struct wm_renderer;
 struct wm_output;
 struct wm_idle_inhibit;
 
+#ifdef WM_HAS_XWAYLAND
+// wlroots defines an enum with some of the same names so strip the prefix
+enum wm_atom_name {
+	WINDOW_TYPE_NORMAL,
+	WINDOW_TYPE_DIALOG,
+	WINDOW_TYPE_UTILITY,
+	WINDOW_TYPE_TOOLBAR,
+	WINDOW_TYPE_MENU,
+    WINDOW_TYPE_DOCK,
+    WM_ATOM_LAST
+};
+#endif
+
 struct wm_server{
     struct wm_config* wm_config;
 
@@ -38,6 +56,7 @@ struct wm_server{
     struct wlr_xdg_decoration_manager_v1* wlr_xdg_decoration_manager;
 #ifdef WM_HAS_XWAYLAND
     struct wlr_xwayland* wlr_xwayland;
+    xcb_atom_t xcb_atoms[WM_ATOM_LAST];
 #endif
     struct wlr_xcursor_manager* wlr_xcursor_manager;
     struct wlr_virtual_keyboard_manager_v1* wlr_virtual_keyboard_manager;

src/wm/wm_server.c

@@ -47,6 +47,14 @@
 #include "wm/wm_view.h"
 #include "wm/wm_drag.h"
 
+static const char *wm_atom_map[WM_ATOM_LAST] = {
+    [WINDOW_TYPE_NORMAL] = "_NET_WM_WINDOW_TYPE_NORMAL",
+    [WINDOW_TYPE_DIALOG] = "_NET_WM_WINDOW_TYPE_DIALOG",
+	[WINDOW_TYPE_UTILITY] = "_NET_WM_WINDOW_TYPE_UTILITY",
+    [WINDOW_TYPE_TOOLBAR] = "_NET_WM_WINDOW_TYPE_TOOLBAR",
+	[WINDOW_TYPE_MENU] = "_NET_WM_WINDOW_TYPE_MENU",
+    [WINDOW_TYPE_DOCK] = "_NET_WM_WINDOW_TYPE_DOCK",
+};
 
 /*
  * Callbacks
@@ -121,6 +129,43 @@ static void handle_new_layer_surface(struct wl_listener* listener, void* data){
 }
 
 #ifdef WM_HAS_XWAYLAND
+static void handle_xwayland_ready(struct wl_listener* listener, void* data)
+{
+    wlr_log(WLR_DEBUG, "Server: XWayland ready");
+
+    struct wm_server* server = wl_container_of(listener, server, xwayland_ready);
+
+    xcb_connection_t *xcb_conn = xcb_connect(NULL, NULL);
+    int err = xcb_connection_has_error(xcb_conn);
+    if (err) {
+        wlr_log(WLR_ERROR, "XCB connect failed: %d", err);
+        return;
+    }
+
+    xcb_intern_atom_cookie_t cookies[WM_ATOM_LAST];
+    for (size_t i = 0; i < WM_ATOM_LAST; i++)
+    {
+        cookies[i] = xcb_intern_atom(xcb_conn, 0, strlen(wm_atom_map[i]), wm_atom_map[i]);
+    }
+    for (size_t i = 0; i < WM_ATOM_LAST; i++)
+    {
+        xcb_generic_error_t *error = NULL;
+		xcb_intern_atom_reply_t *reply =
+			xcb_intern_atom_reply(xcb_conn, cookies[i], &error);
+		if (reply != NULL && error == NULL) {
+			server->xcb_atoms[i] = reply->atom;
+		}
+		free(reply);
+
+		if (error != NULL) {
+			wlr_log(WLR_ERROR, "could not resolve atom %s, X11 error code %d",
+				atom_map[i], error->error_code);
+			free(error);
+			break;
+		}
+    }
+    wm_callback_ready();
+}
 static void handle_new_xwayland_surface(struct wl_listener* listener, void* data){
     wlr_log(WLR_DEBUG, "Server: New xwayland surface");
 
@@ -185,12 +230,6 @@ static void handle_new_xdg_decoration(struct wl_listener* listener, void* data){
 
 }
 
-static void handle_ready(struct wl_listener* listener, void* data){
-    wlr_log(WLR_DEBUG, "Server: Ready");
-
-    wm_callback_ready();
-}
-
 static int callback_timer_handler(void* data){
     struct wm_server* server = data;
 
@@ -352,7 +391,7 @@ void wm_server_init(struct wm_server* server, struct wm_config* config){
         * Due to the unfortunate handling of XWayland forks via SIGUSR1, we need to be sure not
         * to create any threads before the XWayland server is ready
         */
-        server->xwayland_ready.notify = handle_ready;
+        server->xwayland_ready.notify = handle_xwayland_ready;
         wl_signal_add(&server->wlr_xwayland->events.ready, &server->xwayland_ready);
     }
 #endif

src/wm/wm_view_xwayland.c

@@ -1,4 +1,5 @@
 #define _POSIX_C_SOURCE 200809L
+#include "xwayland/xwm.h"
 
 #include <assert.h>
 #include <stdlib.h>
@@ -42,8 +43,28 @@ static void try_to_find_parent(struct wm_view_xwayland* view){
             }
         }
     }
+
+    // pid matching is necessary for override-redirect windows like dropdown menus
+    // and tooltips. sometimes this is conveyed using the window type hint as well
+    bool window_type_needs_parent = view->wlr_xwayland_surface->override_redirect;
+    if(!window_type_needs_parent)
+    {
+        struct wm_server* xwm = view->super.super.wm_server;
+        for (size_t i = 0; i < view->wlr_xwayland_surface->window_type_len; i++) {
+            xcb_atom_t type = view->wlr_xwayland_surface->window_type[i];
+            if (type == xwm->xcb_atoms[WINDOW_TYPE_DOCK] ||
+                type == xwm->xcb_atoms[WINDOW_TYPE_TOOLBAR] ||
+                type == xwm->xcb_atoms[WINDOW_TYPE_MENU] ||
+                type == xwm->xcb_atoms[WINDOW_TYPE_UTILITY] ||
+                type == xwm->xcb_atoms[WINDOW_TYPE_DIALOG])
+                {
+                    window_type_needs_parent = true;
+                    break;
+                }
+        }
+    }
 
-    if(view->wlr_xwayland_surface->pid){
+    if(window_type_needs_parent && view->wlr_xwayland_surface->pid){
         struct wm_content* it;
         wl_list_for_each(it, &view->super.super.wm_server->wm_contents, link){
             if(it->vtable != view->super.super.vtable) continue;
@@ -58,6 +79,7 @@ static void try_to_find_parent(struct wm_view_xwayland* view){
                 parent = it_xwayland;
                 goto Found;
             }
+
         }
     }