Cocoa: Fix clearing of unrelated window style bits

Whenever GLFW changed the window style mask, a new mask was created
from scratch based on the attributes set on the GLFW window object.
This caused us to potentially clear unrelated window style bits.

This was always wrong but became a critical issue when Cocoa began
throwing an exception if an application cleared the
NSWindowStyleMaskFullScreen while the window is in macOS fullscreen.

This commit reworks all style mask editing so it only changes the
relevant bits, preserving all others.

This is only a narrow bug fix to prevent crashes, intended for the
stable branch.  Our interaction with macOS fullscreen is still very
poor.  The next step after this is a set of patches that improve the
interaction between the current API and macOS fullscreen.

Fixes #1886
Fixes #2110

(cherry picked from commit 0d599026d0)

CONTRIBUTORS.md

@@ -83,6 +83,7 @@ video tutorials.
  - heromyth
  - Lucas Hinderberger
  - Paul Holden
+ - Hajime Hoshi
  - Warren Hu
  - Charles Huber
  - Brent Huisman
@@ -147,6 +148,7 @@ video tutorials.
  - James Murphy
  - Julian Møller
  - ndogxj
+ - F. Nedelec
  - Kristian Nielsen
  - Kamil Nowakowski
  - onox

README.md

@@ -137,6 +137,10 @@ information on what to include when reporting a bug.
  - [Cocoa] Bugfix: A connected Apple AirPlay would emit a useless error (#1791)
  - [Cocoa] Bugfix: The EGL and OSMesa libraries were not unloaded on termination
  - [Cocoa] Bugfix: `GLFW_MAXIMIZED` was always true when `GLFW_RESIZABLE` was false
+ - [Cocoa] Bugfix: Changing `GLFW_DECORATED` in macOS fullscreen would abort
+   application (#1886)
+ - [Cocoa] Bugfix: Setting a monitor from macOS fullscreen would abort
+   application (#2110)
  - [X11] Bugfix: The OSMesa libray was not unloaded on termination
  - [X11] Bugfix: A malformed response during selection transfer could cause a segfault
  - [X11] Bugfix: Some calls would reset Xlib to the default error handler (#2108)

src/cocoa_window.m

@@ -35,26 +35,6 @@
 //       having been (according to documentation) added in Mac OS X 10.7
 #define NSWindowCollectionBehaviorFullScreenNone (1 << 9)
 
-// Returns the style mask corresponding to the window settings
-//
-static NSUInteger getStyleMask(_GLFWwindow* window)
-{
-    NSUInteger styleMask = NSWindowStyleMaskMiniaturizable;
-
-    if (window->monitor || !window->decorated)
-        styleMask |= NSWindowStyleMaskBorderless;
-    else
-    {
-        styleMask |= NSWindowStyleMaskTitled |
-                     NSWindowStyleMaskClosable;
-
-        if (window->resizable)
-            styleMask |= NSWindowStyleMaskResizable;
-    }
-
-    return styleMask;
-}
-
 // Returns whether the cursor is in the content area of the specified window
 //
 static GLFWbool cursorInContentArea(_GLFWwindow* window)
@@ -813,9 +793,21 @@ static GLFWbool createNativeWindow(_GLFWwindow* window,
     else
         contentRect = NSMakeRect(0, 0, wndconfig->width, wndconfig->height);
 
+    NSUInteger styleMask = NSWindowStyleMaskMiniaturizable;
+
+    if (window->monitor || !window->decorated)
+        styleMask |= NSWindowStyleMaskBorderless;
+    else
+    {
+        styleMask |= (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+
+        if (window->resizable)
+            styleMask |= NSWindowStyleMaskResizable;
+    }
+
     window->ns.object = [[GLFWWindow alloc]
         initWithContentRect:contentRect
-                  styleMask:getStyleMask(window)
+                  styleMask:styleMask
                     backing:NSBackingStoreBuffered
                       defer:NO];
 
@@ -1244,9 +1236,10 @@ void _glfwPlatformSetWindowMonitor(_GLFWwindow* window,
         {
             const NSRect contentRect =
                 NSMakeRect(xpos, _glfwTransformYNS(ypos + height - 1), width, height);
+            const NSUInteger styleMask = [window->ns.object styleMask];
             const NSRect frameRect =
                 [window->ns.object frameRectForContentRect:contentRect
-                                                 styleMask:getStyleMask(window)];
+                                                 styleMask:styleMask];
 
             [window->ns.object setFrame:frameRect display:YES];
         }
@@ -1263,7 +1256,27 @@ void _glfwPlatformSetWindowMonitor(_GLFWwindow* window,
     // TODO: Solve this in a less terrible way
     _glfwPlatformPollEvents();
 
-    const NSUInteger styleMask = getStyleMask(window);
+    NSUInteger styleMask = [window->ns.object styleMask];
+
+    if (window->monitor)
+    {
+        styleMask &= ~(NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+        styleMask |= NSWindowStyleMaskBorderless;
+    }
+    else
+    {
+        if (window->decorated)
+        {
+            styleMask &= ~NSWindowStyleMaskBorderless;
+            styleMask |= (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+        }
+
+        if (window->resizable)
+            styleMask |= NSWindowStyleMaskResizable;
+        else
+            styleMask &= ~NSWindowStyleMaskResizable;
+    }
+
     [window->ns.object setStyleMask:styleMask];
     // HACK: Changing the style mask can cause the first responder to be cleared
     [window->ns.object makeFirstResponder:window->ns.view];
@@ -1394,10 +1407,10 @@ void _glfwPlatformSetWindowResizable(_GLFWwindow* window, GLFWbool enabled)
 {
     @autoreleasepool {
 
-    [window->ns.object setStyleMask:getStyleMask(window)];
+    const NSUInteger styleMask = [window->ns.object styleMask];
-
     if (enabled)
     {
+        [window->ns.object setStyleMask:(styleMask | NSWindowStyleMaskResizable)];
         const NSWindowCollectionBehavior behavior =
             NSWindowCollectionBehaviorFullScreenPrimary |
             NSWindowCollectionBehaviorManaged;
@@ -1405,6 +1418,7 @@ void _glfwPlatformSetWindowResizable(_GLFWwindow* window, GLFWbool enabled)
     }
     else
     {
+        [window->ns.object setStyleMask:(styleMask & ~NSWindowStyleMaskResizable)];
         const NSWindowCollectionBehavior behavior =
             NSWindowCollectionBehaviorFullScreenNone;
         [window->ns.object setCollectionBehavior:behavior];
@@ -1416,8 +1430,22 @@ void _glfwPlatformSetWindowResizable(_GLFWwindow* window, GLFWbool enabled)
 void _glfwPlatformSetWindowDecorated(_GLFWwindow* window, GLFWbool enabled)
 {
     @autoreleasepool {
-    [window->ns.object setStyleMask:getStyleMask(window)];
+
+    NSUInteger styleMask = [window->ns.object styleMask];
+    if (enabled)
+    {
+        styleMask |= (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+        styleMask &= ~NSWindowStyleMaskBorderless;
+    }
+    else
+    {
+        styleMask |= NSWindowStyleMaskBorderless;
+        styleMask &= ~(NSWindowStyleMaskTitled | NSWindowStyleMaskClosable);
+    }
+
+    [window->ns.object setStyleMask:styleMask];
     [window->ns.object makeFirstResponder:window->ns.view];
+
     } // autoreleasepool
 }