Undecorated border fix + customizable caption/bordersize

Added functions to set caption area and resize border size for undecorated windows. Implemented this functionality for win32 using WM_NCHITTEST and WM_NCCALCSIZE.

include/GLFW/glfw3.h

@@ -3884,7 +3884,7 @@ GLFWAPI void glfwFocusWindow(GLFWwindow* window);
  *
  *  @ingroup window
  */
-GLFWAPI void glfwDragWindow(GLFWwindow* handle);
+GLFWAPI void glfwDragWindow(GLFWwindow* window);
 
 /*! @brief Starts a resize operation with the specified window.
  *
@@ -3919,6 +3919,41 @@ GLFWAPI void glfwDragWindow(GLFWwindow* handle);
  */
 GLFWAPI void glfwResizeWindow(GLFWwindow* window, int border);
 
+/*! @brief Sets the caption area for the specified window.
+ *
+ *  This function sets the rectangle for the caption to drag the undecorated window.
+ *
+ *  @param[in] window The window to set the caption area for.
+ *  @param[in] offsetX The x offset from the top left of the window.
+ *  @param[in] offsetY The y offset from the top left of the window.
+ *  @param[in] sizeX The x size of the caption area.
+ *  @param[in] sizeY The y size of the caption area.
+ *  @thread_safety This function must only be called from the main thread.
+ *
+ *  @sa @ref window_caption_area
+ *
+ *  @since Added in version 3.4.
+ *
+ *  @ingroup window
+ */
+GLFWAPI void glfwWindowSetCaptionArea(GLFWwindow* window, int offsetX, int offsetY, int sizeX, int sizeY);
+
+/*! @brief Sets the resize border size for the specified window.
+ *
+ *  This function sets the size of border where to start the resize operation.
+ *
+ *  @param[in] window The window to set the caption area for.
+ *  @param[in] size The size of the border.
+ *  @thread_safety This function must only be called from the main thread.
+ *
+ *  @sa @ref window_border_size
+ *
+ *  @since Added in version 3.4.
+ *
+ *  @ingroup window
+ */
+GLFWAPI void glfwWindowSetResizeBorderSize(GLFWwindow* window, int size);
+
 /*! @brief Requests user attention to the specified window.
  *
  *  This function requests user attention to the specified window.  On

src/internal.h

@@ -539,6 +539,11 @@ struct _GLFWwindow
     int                 maxwidth, maxheight;
     int                 numer, denom;
 
+    // Caption for undecorated window dragging functionality
+    int                 captionOffsetX, captionOffsetY;
+    int                 captionSizeX, captionSizeY;
+    int                 resizeBorderSize;
+
     GLFWbool            stickyKeys;
     GLFWbool            stickyMouseButtons;
     GLFWbool            lockKeyMods;

src/win32_window.c

@@ -62,7 +62,7 @@ static DWORD getWindowStyle(const _GLFWwindow* window)
             style |= WS_POPUP;
 
             if (window->resizable)
-                style |= WS_THICKFRAME;
+                style |= WS_MAXIMIZEBOX | WS_THICKFRAME;
         }
     }
 
@@ -1248,6 +1248,80 @@ static LRESULT CALLBACK windowProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM l
             DragFinish(drop);
             return 0;
         }
+
+        case WM_NCCALCSIZE:
+        {
+            if (wParam && !window->decorated)
+                return 0;
+
+            break;
+        }
+
+        case WM_NCHITTEST:
+        {
+            if (!window->decorated) 
+            {
+                POINT cursor;
+                cursor.x = GET_X_LPARAM(lParam);
+                cursor.y = GET_Y_LPARAM(lParam);
+
+                RECT rect;
+                if (!GetWindowRect(hWnd, &rect))
+                    return HTNOWHERE;
+
+                POINT border = { 0, 0 };
+                if (window->resizable)
+                {
+                    border.x = window->resizeBorderSize;
+                    border.y = window->resizeBorderSize;
+                }
+
+                const int captionOffsetX    = window->captionOffsetX != GLFW_DONT_CARE ? window->captionOffsetX : 0;
+                const int captionOffsetY    = window->captionOffsetY != GLFW_DONT_CARE ? window->captionOffsetY : 0;
+                const int captionSizeX      = window->captionSizeX != GLFW_DONT_CARE ? window->captionSizeX : rect.right - rect.left;
+                const int captionSizeY      = window->captionSizeY;
+
+                const int clientAreaLeft    = rect.left + border.x;
+                const int clientAreaRight   = rect.right - border.x;
+                const int clientAreaTop     = rect.top + border.y;
+                const int clientAreaBottom  = rect.bottom - border.y;
+
+                const int cursorInCaption =
+                    cursor.x > clientAreaLeft + captionOffsetX &&
+                    cursor.x < clientAreaLeft + captionOffsetX + captionSizeX &&
+                    cursor.y > clientAreaTop  + captionOffsetY &&
+                    cursor.y < clientAreaTop  + captionOffsetY + captionSizeY;
+
+                enum region_mask {
+                    client = 0b0000,
+                    left = 0b0001,
+                    right = 0b0010,
+                    top = 0b0100,
+                    bottom = 0b1000,
+                };
+
+                const int result =
+                    left *      (cursor.x <  clientAreaLeft) |
+                    right *     (cursor.x >= clientAreaRight)|
+                    top *       (cursor.y <  clientAreaTop)  |
+                    bottom *    (cursor.y >= clientAreaBottom);
+
+                switch (result) 
+                {
+                    case left: return HTLEFT;
+                    case right: return HTRIGHT;
+                    case top: return HTTOP;
+                    case bottom: return HTBOTTOM;
+                    case top | left: return HTTOPLEFT;
+                    case top | right: return HTTOPRIGHT;
+                    case bottom | left: return HTBOTTOMLEFT;
+                    case bottom | right: return HTBOTTOMRIGHT;
+                    case client: return cursorInCaption ? HTCAPTION : HTCLIENT;
+                    default: return HTNOWHERE;
+                }
+            }
+            break;
+        }
     }
 
     return DefWindowProcW(hWnd, uMsg, wParam, lParam);

src/window.c

@@ -245,6 +245,13 @@ GLFWAPI GLFWwindow* glfwCreateWindow(int width, int height,
     window->numer       = GLFW_DONT_CARE;
     window->denom       = GLFW_DONT_CARE;
 
+    window->captionOffsetX  = GLFW_DONT_CARE;
+    window->captionOffsetY  = GLFW_DONT_CARE;
+    window->captionSizeX    = GLFW_DONT_CARE;
+    window->captionSizeY    = 16;
+
+    window->resizeBorderSize = 4;
+
     if (!_glfw.platform.createWindow(window, &wndconfig, &ctxconfig, &fbconfig))
     {
         glfwDestroyWindow((GLFWwindow*) window);
@@ -868,6 +875,35 @@ GLFWAPI void glfwResizeWindow(GLFWwindow* handle, int border)
     _glfw.platform.resizeWindow(window, border);
 }
 
+GLFWAPI void glfwWindowSetCaptionArea(GLFWwindow* handle, int offsetX, int offsetY, int sizeX, int sizeY)
+{
+    _GLFWwindow* window = (_GLFWwindow*)handle;
+    assert(window != NULL);
+
+    _GLFW_REQUIRE_INIT();
+
+    if (offsetX < 0 || offsetY < 0 || sizeX < 1 || sizeY < 1)
+        return;
+
+    window->captionOffsetX = offsetX;
+    window->captionOffsetY = offsetY;
+    window->captionSizeX = sizeX;
+    window->captionSizeY = sizeY;
+}
+
+GLFWAPI void glfwWindowSetResizeBorderSize(GLFWwindow* handle, int size)
+{
+    _GLFWwindow* window = (_GLFWwindow*)handle;
+    assert(window != NULL);
+
+    _GLFW_REQUIRE_INIT();
+
+    if (size < 1)
+        return;
+
+    window->resizeBorderSize = size;
+}
+
 GLFWAPI int glfwGetWindowAttrib(GLFWwindow* handle, int attrib)
 {
     _GLFWwindow* window = (_GLFWwindow*) handle;