Rounded corners for legacy glx backend

Authored-by: bhagwan <bhagwan@disroot.org>
Authored-by: Samuel Hand <samuel.d.hand@gmail.com>

src/opengl.c

@@ -95,6 +95,14 @@ bool glx_init(session_t *ps, bool need_render) {
 			ppass->unifm_offset_x = -1;
 			ppass->unifm_offset_y = -1;
 		}
+
+		ps->psglx->round_passes = ccalloc(1, glx_round_pass_t);
+		glx_round_pass_t *ppass = ps->psglx->round_passes;
+		ppass->unifm_radius = -1;
+		ppass->unifm_texcoord = -1;
+		ppass->unifm_texsize = -1;
+		ppass->unifm_borderw = -1;
+		ppass->unifm_resolution = -1;
 	}
 
 	glx_session_t *psglx = ps->psglx;
@@ -232,13 +240,24 @@ void glx_destroy(session_t *ps) {
 	// Free GLSL shaders/programs
 	for (int i = 0; i < ps->o.blur_kernel_count; ++i) {
 		glx_blur_pass_t *ppass = &ps->psglx->blur_passes[i];
-		if (ppass->frag_shader)
+		if (ppass->frag_shader) {
 			glDeleteShader(ppass->frag_shader);
-		if (ppass->prog)
+		}
+		if (ppass->prog) {
 			glDeleteProgram(ppass->prog);
 		}
+	}
 	free(ps->psglx->blur_passes);
 
+	glx_round_pass_t *ppass = ps->psglx->round_passes;
+	if (ppass->frag_shader) {
+		glDeleteShader(ppass->frag_shader);
+	}
+	if (ppass->prog) {
+		glDeleteProgram(ppass->prog);
+	}
+	free(ps->psglx->round_passes);
+
 	glx_free_prog_main(&ps->glx_prog_win);
 
 	gl_check_err();
@@ -415,6 +434,145 @@ bool glx_init_blur(session_t *ps) {
 	return true;
 }
 
+/**
+ * Initialize GLX rounded corners filter.
+ */
+bool glx_init_rounded_corners(session_t *ps) {
+	char *lc_numeric_old = strdup(setlocale(LC_NUMERIC, NULL));
+	// Enforce LC_NUMERIC locale "C" here to make sure decimal point is sane
+	// Thanks to hiciu for reporting.
+	setlocale(LC_NUMERIC, "C");
+
+	static const char *FRAG_SHADER =
+	    "#version 110\n"
+	    "%s"        // extensions
+	    "uniform float u_radius;\n"
+	    "uniform float u_borderw;\n"
+	    "uniform int u_is_focused;\n"
+	    "uniform vec2 u_texcoord;\n"
+	    "uniform vec2 u_texsize;\n"
+	    "uniform vec2 u_resolution;\n"
+	    "uniform %s tex_scr;\n"        // sampler2D | sampler2DRect
+	    "\n"
+	    "// https://www.shadertoy.com/view/ltS3zW\n"
+	    "float RectSDF(vec2 p, vec2 b, float r) {\n"
+	    "  vec2 d = abs(p) - b + vec2(r);\n"
+	    "  return min(max(d.x, d.y), 0.0) + length(max(d, 0.0)) - r;\n"
+	    "}\n\n"
+	    "// https://www.shadertoy.com/view/ldfSDj\n"
+	    "float udRoundBox( vec2 p, vec2 b, float r )\n"
+	    "{\n"
+	    "  return length(max(abs(p)-b+r,0.0))-r;\n"
+	    "}\n\n"
+	    "void main()\n"
+	    "{\n"
+	    "  vec2 coord = vec2(u_texcoord.x, "
+	    "u_resolution.y-u_texsize.y-u_texcoord.y);\n"
+	    "  vec4 u_v4SrcColor = %s(tex_scr, vec2(gl_TexCoord[0].st));\n"
+	    "  float u_fRadiusPx = u_radius;\n"
+	    "  float u_fHalfBorderThickness = 0.0;\n"
+	    "  vec4 u_v4BorderColor = vec4(1.0, 0.0, 0.0, 1.0);\n"
+	    "  vec4 u_v4FillColor = vec4(0.0, 0.0, 0.0, 0.0);\n"
+	    "  vec4 v4FromColor = u_v4BorderColor;	//Always the border "
+	    "color. If no border, this still should be set\n"
+	    "  vec4 v4ToColor = u_v4SrcColor;		//Outside color is the "
+	    "background texture\n"
+	    "\n"
+	    "  vec2 u_v2HalfShapeSizePx = u_texsize/2.0 - "
+	    "vec2(u_fHalfBorderThickness);\n"
+	    "  vec2 v_v2CenteredPos = (gl_FragCoord.xy - u_texsize.xy / 2.0 - "
+	    "coord);\n"
+	    "\n"
+	    "  float fDist = RectSDF(v_v2CenteredPos, u_v2HalfShapeSizePx, "
+	    "u_fRadiusPx - u_fHalfBorderThickness);\n"
+	    "  if (u_fHalfBorderThickness > 0.0) {\n"
+	    "    if (fDist < 0.0) {\n"
+	    "      v4ToColor = u_v4FillColor;\n"
+	    "    }\n"
+	    "    fDist = abs(fDist) - u_fHalfBorderThickness;\n"
+	    "  } else {\n"
+	    "    v4FromColor = u_v4FillColor;\n"
+	    "  }\n"
+	    "  float fBlendAmount = clamp(fDist + 0.5, 0.0, 1.0);\n"
+	    "  vec4 c = mix(v4FromColor, v4ToColor, fBlendAmount);\n"
+	    "\n"
+	    "  // final color\n"
+	    "  gl_FragColor = c;\n"
+	    "\n"
+	    "}\n";
+
+	const bool use_texture_rect = !ps->psglx->has_texture_non_power_of_two;
+	const char *sampler_type = (use_texture_rect ? "sampler2DRect" : "sampler2D");
+	const char *texture_func = (use_texture_rect ? "texture2DRect" : "texture2D");
+	char *extension = NULL;
+	if (use_texture_rect) {
+		mstrextend(&extension, "#extension GL_ARB_texture_rectangle : "
+		                       "require\n");
+	}
+	if (!extension) {
+		extension = strdup("");
+	}
+
+	bool success = false;
+	// Build rounded corners shader
+	auto ppass = ps->psglx->round_passes;
+	auto len = strlen(FRAG_SHADER) + strlen(extension) + strlen(sampler_type) +
+	           strlen(texture_func) + 1;
+	char *shader_str = ccalloc(len, char);
+
+	char *pc = shader_str;
+	sprintf(pc, FRAG_SHADER, extension, sampler_type, texture_func);
+	pc += strlen(pc);
+	assert(strlen(shader_str) < len);
+
+	log_debug("Generated rounded corners shader:\n%s\n", shader_str);
+
+	ppass->frag_shader = gl_create_shader(GL_FRAGMENT_SHADER, shader_str);
+	free(shader_str);
+
+	if (!ppass->frag_shader) {
+		log_error("Failed to create rounded corners fragment shader.");
+		goto out;
+	}
+
+	// Build program
+	ppass->prog = gl_create_program(&ppass->frag_shader, 1);
+	if (!ppass->prog) {
+		log_error("Failed to create GLSL program.");
+		goto out;
+	}
+
+	// Get uniform addresses
+#define P_GET_UNIFM_LOC(name, target)                                                    \
+	{                                                                                \
+		ppass->target = glGetUniformLocation(ppass->prog, name);                 \
+		if (ppass->target < 0) {                                                 \
+			log_debug("Failed to get location of rounded corners uniform "   \
+			          "'" name "'. Might be troublesome.");                  \
+		}                                                                        \
+	}
+	P_GET_UNIFM_LOC("u_radius", unifm_radius);
+	P_GET_UNIFM_LOC("u_texcoord", unifm_texcoord);
+	P_GET_UNIFM_LOC("u_texsize", unifm_texsize);
+	P_GET_UNIFM_LOC("u_borderw", unifm_borderw);
+	P_GET_UNIFM_LOC("u_is_focused", unifm_is_focused);
+	P_GET_UNIFM_LOC("u_resolution", unifm_resolution);
+#undef P_GET_UNIFM_LOC
+
+	success = true;
+
+out:
+	free(extension);
+
+	// Restore LC_NUMERIC
+	setlocale(LC_NUMERIC, lc_numeric_old);
+	free(lc_numeric_old);
+
+	gl_check_err();
+
+	return success;
+}
+
 /**
  * Load a GLSL main program from shader strings.
  */
@@ -449,6 +607,88 @@ bool glx_load_prog_main(const char *vshader_str, const char *fshader_str,
 	return true;
 }
 
+static inline void glx_copy_region_to_tex(session_t *ps, GLenum tex_tgt, int basex,
+                                          int basey, int dx, int dy, int width, int height) {
+	if (width > 0 && height > 0) {
+		glCopyTexSubImage2D(tex_tgt, 0, dx - basex, dy - basey, dx,
+		                    ps->root_height - dy - height, width, height);
+	}
+}
+
+static inline GLuint glx_gen_texture(GLenum tex_tgt, int width, int height) {
+	GLuint tex = 0;
+	glGenTextures(1, &tex);
+	if (!tex) {
+		return 0;
+	}
+	glEnable(tex_tgt);
+	glBindTexture(tex_tgt, tex);
+	glTexParameteri(tex_tgt, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+	glTexParameteri(tex_tgt, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+	glTexParameteri(tex_tgt, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+	glTexParameteri(tex_tgt, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+	glTexImage2D(tex_tgt, 0, GL_RGB, width, height, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
+	glBindTexture(tex_tgt, 0);
+
+	return tex;
+}
+
+/**
+ * Bind an OpenGL texture and fill it with pixel data from back buffer
+ */
+bool glx_bind_texture(session_t *ps attr_unused, glx_texture_t **pptex, int x, int y,
+                      int width, int height) {
+	if (ps->o.backend != BKEND_GLX && ps->o.backend != BKEND_XR_GLX_HYBRID) {
+		return true;
+	}
+
+	glx_texture_t *ptex = *pptex;
+
+	// log_trace("Copying xy(%d %d) wh(%d %d)", x, y, width, height);
+
+	// Release texture if parameters are inconsistent
+	if (ptex && ptex->texture && (ptex->width != width || ptex->height != height)) {
+		free_texture(ps, &ptex);
+	}
+
+	// Allocate structure
+	if (!ptex) {
+		ptex = ccalloc(1, glx_texture_t);
+		*pptex = ptex;
+
+		ptex->width = width;
+		ptex->height = height;
+		ptex->target = GL_TEXTURE_RECTANGLE;
+		if (ps->psglx->has_texture_non_power_of_two) {
+			ptex->target = GL_TEXTURE_2D;
+		}
+	}
+
+	// Create texture
+	if (!ptex->texture) {
+		ptex->texture = glx_gen_texture(ptex->target, width, height);
+	}
+	if (!ptex->texture) {
+		log_error("Failed to allocate texture.");
+		return false;
+	}
+
+	// Read destination pixels into a texture
+	glEnable(ptex->target);
+	glBindTexture(ptex->target, ptex->texture);
+	if (width > 0 && height > 0) {
+		glx_copy_region_to_tex(ps, ptex->target, x, y, x, y, width, height);
+	}
+
+	// Cleanup
+	glBindTexture(ptex->target, 0);
+	glDisable(ptex->target);
+
+	gl_check_err();
+
+	return true;
+}
+
 /**
  * Bind a X pixmap to an OpenGL texture.
  */
@@ -662,30 +902,6 @@ void glx_set_clip(session_t *ps, const region_t *reg) {
                                                                                          \
 	pixman_region32_fini(&reg_new);
 
-static inline GLuint glx_gen_texture(GLenum tex_tgt, int width, int height) {
-	GLuint tex = 0;
-	glGenTextures(1, &tex);
-	if (!tex)
-		return 0;
-	glEnable(tex_tgt);
-	glBindTexture(tex_tgt, tex);
-	glTexParameteri(tex_tgt, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-	glTexParameteri(tex_tgt, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-	glTexParameteri(tex_tgt, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-	glTexParameteri(tex_tgt, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-	glTexImage2D(tex_tgt, 0, GL_RGB, width, height, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
-	glBindTexture(tex_tgt, 0);
-
-	return tex;
-}
-
-static inline void glx_copy_region_to_tex(session_t *ps, GLenum tex_tgt, int basex,
-                                          int basey, int dx, int dy, int width, int height) {
-	if (width > 0 && height > 0)
-		glCopyTexSubImage2D(tex_tgt, 0, dx - basex, dy - basey, dx,
-		                    ps->root_height - dy - height, width, height);
-}
-
 /**
  * Blur contents in a particular region.
  *
@@ -889,6 +1105,118 @@ glx_blur_dst_end:
 	return ret;
 }
 
+bool glx_round_corners_dst(session_t *ps, struct managed_win *w, const glx_texture_t *ptex,
+                           int dx, int dy, int width, int height, float z, float cr,
+                           const region_t *reg_tgt attr_unused) {
+	assert(ps->psglx->round_passes->prog);
+	bool ret = false;
+
+	// log_warn("dxy(%d, %d) wh(%d %d) rwh(%d %d) b(%d), f(%d)",
+	//	dx, dy, width, height, ps->root_width, ps->root_height, w->g.border_width,
+	// w->focused);
+
+	int mdx = dx, mdy = dy, mwidth = width, mheight = height;
+	log_trace("%d, %d, %d, %d", mdx, mdy, mwidth, mheight);
+
+	{
+		const glx_round_pass_t *ppass = ps->psglx->round_passes;
+		assert(ppass->prog);
+
+		glEnable(GL_BLEND);
+		glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
+
+		glUseProgram(ppass->prog);
+
+		// If caller specified a texture use it as source
+		log_trace("ptex: %p wh(%d %d) %d %d", ptex, ptex->width, ptex->height,
+		          ptex->target, ptex->texture);
+
+		glActiveTexture(GL_TEXTURE0);
+		glBindTexture(ptex->target, ptex->texture);
+		GLint loc_sampler = glGetUniformLocation(ppass->prog, "tex_scr");
+		glUniform1i(loc_sampler, (GLint)0);
+
+		if (ppass->unifm_radius >= 0) {
+			glUniform1f(ppass->unifm_radius, cr);
+		}
+		if (ppass->unifm_texcoord >= 0) {
+			glUniform2f(ppass->unifm_texcoord, (float)dx, (float)dy);
+		}
+		if (ppass->unifm_texsize >= 0) {
+			glUniform2f(ppass->unifm_texsize, (float)mwidth, (float)mheight);
+		}
+		if (ppass->unifm_borderw >= 0) {
+			glUniform1f(ppass->unifm_borderw, w->g.border_width);
+		}
+		if (ppass->unifm_is_focused >= 0) {
+			glUniform1i(ppass->unifm_is_focused, w->focused);
+		}
+		if (ppass->unifm_resolution >= 0) {
+			glUniform2f(ppass->unifm_resolution, (float)ps->root_width,
+			            (float)ps->root_height);
+		}
+
+		// Painting
+		{
+			P_PAINTREG_START(crect) {
+				// texture-local coordinates
+				auto rx = (GLfloat)(crect.x1 - dx);
+				auto ry = (GLfloat)(crect.y1 - dy);
+				auto rxe = rx + (GLfloat)(crect.x2 - crect.x1);
+				auto rye = ry + (GLfloat)(crect.y2 - crect.y1);
+				if (GL_TEXTURE_2D == ptex->target) {
+					rx = rx / (GLfloat)width;
+					ry = ry / (GLfloat)height;
+					rxe = rxe / (GLfloat)width;
+					rye = rye / (GLfloat)height;
+				}
+
+				// coordinates for the texture in the target
+				auto rdx = (GLfloat)crect.x1;
+				auto rdy = (GLfloat)(ps->root_height - crect.y1);
+				auto rdxe = (GLfloat)rdx + (GLfloat)(crect.x2 - crect.x1);
+				auto rdye = (GLfloat)rdy - (GLfloat)(crect.y2 - crect.y1);
+
+				// Invert Y if needed, this may not work as expected,
+				// though. I don't have such a FBConfig to test with.
+				ry = 1.0f - ry;
+				rye = 1.0f - rye;
+
+				// log_trace("Rect %d (i:%d): %f, %f, %f, %f -> %f, %f,
+				// %f, %f", 	ri ,ptex ? ptex->y_inverted : -1, rx, ry,
+				// rxe,
+				// rye, rdx, rdy, rdxe, rdye);
+
+				glTexCoord2f(rx, ry);
+				glVertex3f(rdx, rdy, z);
+
+				glTexCoord2f(rxe, ry);
+				glVertex3f(rdxe, rdy, z);
+
+				glTexCoord2f(rxe, rye);
+				glVertex3f(rdxe, rdye, z);
+
+				glTexCoord2f(rx, rye);
+				glVertex3f(rdx, rdye, z);
+			}
+			P_PAINTREG_END();
+		}
+
+		glUseProgram(0);
+		glDisable(GL_BLEND);
+	}
+
+	ret = true;
+
+	glBindTexture(ptex->target, 0);
+	glDisable(ptex->target);
+	glDisable(GL_BLEND);
+
+	gl_check_err();
+
+	return ret;
+}
+
 bool glx_dim_dst(session_t *ps, int dx, int dy, int width, int height, int z,
                  GLfloat factor, const region_t *reg_tgt) {
 	// It's possible to dim in glx_render(), but it would be over-complicated
@@ -1038,7 +1366,8 @@ bool glx_render(session_t *ps, const glx_texture_t *ptex, int x, int y, int dx,
 		if (pprogram->unifm_tex >= 0)
 			glUniform1i(pprogram->unifm_tex, 0);
 		if (pprogram->unifm_time >= 0)
-			glUniform1f(pprogram->unifm_time, (float)ts.tv_sec * 1000.0f + (float)ts.tv_nsec / 1.0e6f);
+			glUniform1f(pprogram->unifm_time, (float)ts.tv_sec * 1000.0f +
+			                                      (float)ts.tv_nsec / 1.0e6f);
 	}
 
 	// log_trace("Draw: %d, %d, %d, %d -> %d, %d (%d, %d) z %d", x, y, width, height,
@@ -1055,7 +1384,7 @@ bool glx_render(session_t *ps, const glx_texture_t *ptex, int x, int y, int dx,
 	// Painting
 	{
 		P_PAINTREG_START(crect) {
-			// XXX explain these variables
+			// texture-local coordinates
 			auto rx = (GLfloat)(crect.x1 - dx + x);
 			auto ry = (GLfloat)(crect.y1 - dy + y);
 			auto rxe = rx + (GLfloat)(crect.x2 - crect.x1);
@@ -1068,6 +1397,8 @@ bool glx_render(session_t *ps, const glx_texture_t *ptex, int x, int y, int dx,
 				rxe = rxe / (GLfloat)ptex->width;
 				rye = rye / (GLfloat)ptex->height;
 			}
+
+			// coordinates for the texture in the target
 			GLint rdx = crect.x1;
 			GLint rdy = ps->root_height - crect.y1;
 			GLint rdxe = rdx + (crect.x2 - crect.x1);

src/opengl.h

@@ -40,6 +40,25 @@ typedef struct {
 	GLint unifm_factor_center;
 } glx_blur_pass_t;
 
+typedef struct {
+	/// Fragment shader for rounded corners.
+	GLuint frag_shader;
+	/// GLSL program for rounded corners.
+	GLuint prog;
+	/// Location of uniform "radius" in rounded-corners GLSL program.
+	GLint unifm_radius;
+	/// Location of uniform "texcoord" in rounded-corners GLSL program.
+	GLint unifm_texcoord;
+	/// Location of uniform "texsize" in rounded-corners GLSL program.
+	GLint unifm_texsize;
+	/// Location of uniform "borderw" in rounded-corners GLSL program.
+	GLint unifm_borderw;
+	/// Location of uniform "is_focused" in rounded-corners GLSL program.
+	GLint unifm_is_focused;
+	/// Location of uniform "resolution" in rounded-corners GLSL program.
+	GLint unifm_resolution;
+} glx_round_pass_t;
+
 /// Structure containing GLX-dependent data for a session.
 typedef struct glx_session {
 	// === OpenGL related ===
@@ -50,6 +69,7 @@ typedef struct glx_session {
 	/// Current GLX Z value.
 	int z;
 	glx_blur_pass_t *blur_passes;
+	glx_round_pass_t *round_passes;
 } glx_session_t;
 
 /// @brief Wrapper of a binded GLX texture.
@@ -81,6 +101,8 @@ void glx_on_root_change(session_t *ps);
 
 bool glx_init_blur(session_t *ps);
 
+bool glx_init_rounded_corners(session_t *ps);
+
 #ifdef CONFIG_OPENGL
 bool glx_load_prog_main(const char *vshader_str, const char *fshader_str,
                         glx_prog_main_t *pprogram);
@@ -91,6 +113,8 @@ bool glx_bind_pixmap(session_t *ps, glx_texture_t **pptex, xcb_pixmap_t pixmap,
 
 void glx_release_pixmap(session_t *ps, glx_texture_t *ptex);
 
+bool glx_bind_texture(session_t *ps, glx_texture_t **pptex, int x, int y, int width, int height);
+
 void glx_paint_pre(session_t *ps, region_t *preg) attr_nonnull(1, 2);
 
 /**
@@ -105,6 +129,10 @@ void glx_set_clip(session_t *ps, const region_t *reg);
 bool glx_blur_dst(session_t *ps, int dx, int dy, int width, int height, float z,
                   GLfloat factor_center, const region_t *reg_tgt, glx_blur_cache_t *pbc);
 
+bool glx_round_corners_dst(session_t *ps, struct managed_win *w,
+                           const glx_texture_t *ptex, int dx, int dy, int width,
+                           int height, float z, float cr, const region_t *reg_tgt);
+
 GLuint glx_create_shader(GLenum shader_type, const char *shader_str);
 
 GLuint glx_create_program(const GLuint *const shaders, int nshaders);
@@ -178,8 +206,9 @@ static inline void free_texture(session_t *ps, glx_texture_t **pptex) {
 	glx_texture_t *ptex = *pptex;
 
 	// Quit if there's nothing
-	if (!ptex)
+	if (!ptex) {
 		return;
+	}
 
 	glx_release_pixmap(ps, ptex);
 
@@ -188,7 +217,6 @@ static inline void free_texture(session_t *ps, glx_texture_t **pptex) {
 	// Free structure itself
 	free(ptex);
 	*pptex = NULL;
-	assert(!*pptex);
 }
 
 /**
@@ -210,5 +238,6 @@ static inline void free_win_res_glx(session_t *ps, struct managed_win *w) {
 	free_paint_glx(ps, &w->shadow_paint);
 #ifdef CONFIG_OPENGL
 	free_glx_bc(ps, &w->glx_blur_cache);
+	free_texture(ps, &w->glx_texture_bg);
 #endif
 }

src/options.c

@@ -1002,9 +1002,8 @@ bool get_cfg(options_t *opt, int argc, char *const *argv, bool shadow_enable,
 		         "properly under X Render backend.");
 	}
 
-	if (opt->corner_radius > 0 &&
+	if (opt->corner_radius > 0 && opt->experimental_backends) {
-	    (opt->backend != BKEND_XRENDER || opt->experimental_backends)) {
+		log_warn("Rounded corner is only supported on legacy backends, it "
-		log_warn("Rounded corner is only supported on legacy xrender backend, it "
 		         "will be disabled");
 		opt->corner_radius = 0;
 	}

src/render.c

@@ -746,25 +746,33 @@ win_paint_shadow(session_t *ps, struct managed_win *w, region_t *reg_paint) {
 	bool should_clip =
 	    (w->corner_radius > 0) && (!ps->o.wintype_option[w->window_type].full_shadow);
 	if (should_clip) {
+		if (ps->o.backend == BKEND_XRENDER || ps->o.backend == BKEND_XR_GLX_HYBRID) {
 			uint32_t max_ntraps = to_u32_checked(w->corner_radius);
 			xcb_render_trapezoid_t traps[4 * max_ntraps + 3];
 			uint32_t n = make_rounded_window_shape(
 			    traps, max_ntraps, w->corner_radius, w->widthb, w->heightb);
 
-		td = x_create_picture_with_standard(ps->c, ps->root, w->widthb, w->heightb,
+			td = x_create_picture_with_standard(
+			    ps->c, ps->root, w->widthb, w->heightb,
 			    XCB_PICT_STANDARD_ARGB_32, 0, 0);
-		xcb_render_color_t trans = {.red = 0, .blue = 0, .green = 0, .alpha = 0};
+			xcb_render_color_t trans = {
+			    .red = 0, .blue = 0, .green = 0, .alpha = 0};
 			const xcb_rectangle_t rect = {.x = 0,
 			                              .y = 0,
 			                              .width = to_u16_checked(w->widthb),
 			                              .height = to_u16_checked(w->heightb)};
-		xcb_render_fill_rectangles(ps->c, XCB_RENDER_PICT_OP_SRC, td, trans, 1, &rect);
+			xcb_render_fill_rectangles(ps->c, XCB_RENDER_PICT_OP_SRC, td,
+			                           trans, 1, &rect);
 
 			auto solid = solid_picture(ps->c, ps->root, false, 1, 0, 0, 0);
-		xcb_render_trapezoids(ps->c, XCB_RENDER_PICT_OP_OVER, solid, td,
+			xcb_render_trapezoids(
-		                      x_get_pictfmt_for_standard(ps->c, XCB_PICT_STANDARD_A_8),
+			    ps->c, XCB_RENDER_PICT_OP_OVER, solid, td,
-		                      0, 0, n, traps);
+			    x_get_pictfmt_for_standard(ps->c, XCB_PICT_STANDARD_A_8), 0,
+			    0, n, traps);
 			xcb_render_free_picture(ps->c, solid);
+		} else {
+			// Not implemented
+		}
 	}
 
 	clip_t clip = {
@@ -1112,6 +1120,18 @@ void paint_all(session_t *ps, struct managed_win *t, bool ignore_damage) {
 
 		if (pixman_region32_not_empty(&reg_tmp)) {
 			set_tgt_clip(ps, &reg_tmp);
+
+#ifdef CONFIG_OPENGL
+			// If rounded corners backup the region first
+			if (w->corner_radius > 0 && ps->o.backend == BKEND_GLX) {
+				const int16_t x = w->g.x;
+				const int16_t y = w->g.y;
+				const auto wid = to_u16_checked(w->widthb);
+				const auto hei = to_u16_checked(w->heightb);
+				glx_bind_texture(ps, &w->glx_texture_bg, x, y, wid, hei);
+			}
+#endif
+
 			// Blur window background
 			if (w->blur_background &&
 			    (w->mode == WMODE_TRANS ||
@@ -1121,6 +1141,19 @@ void paint_all(session_t *ps, struct managed_win *t, bool ignore_damage) {
 
 			// Painting the window
 			paint_one(ps, w, &reg_tmp);
+
+#ifdef CONFIG_OPENGL
+			// Rounded corners for XRender is implemented inside render()
+			// Round window corners
+			if (w->corner_radius > 0 && ps->o.backend == BKEND_GLX) {
+				const auto wid = to_u16_checked(w->widthb);
+				const auto hei = to_u16_checked(w->heightb);
+				glx_round_corners_dst(ps, w, w->glx_texture_bg, w->g.x,
+				                      w->g.y, wid, hei,
+				                      (float)ps->psglx->z - 0.5f,
+				                      (float)w->corner_radius, reg_paint);
+			}
+#endif
 		}
 	}
 
@@ -1374,6 +1407,18 @@ bool init_render(session_t *ps) {
 			return false;
 		}
 	}
+
+	// Initialize our rounded corners fragment shader
+	if (ps->o.corner_radius > 0 && ps->o.backend == BKEND_GLX) {
+#ifdef CONFIG_OPENGL
+		if (!glx_init_rounded_corners(ps)) {
+			log_error("Failed to init rounded corners shader.");
+			return false;
+		}
+#else
+		assert(false);
+#endif
+	}
 	return true;
 }
 

src/win.h

@@ -270,6 +270,8 @@ struct managed_win {
 #ifdef CONFIG_OPENGL
 	/// Textures and FBO background blur use.
 	glx_blur_cache_t glx_blur_cache;
+	/// Background texture of the window
+	glx_texture_t *glx_texture_bg;
 #endif
 };