backend: gl: add dither

Add bayer ordered dithering when presenting to screen. Reduce banding
when using a strong blur. Also use 16-bit intermediary textures to
preserve precision in the rendering pipeline.

Related: #602

Signed-off-by: Yuxuan Shui <yshuiv7@gmail.com>

src/backend/gl/blur.c

@@ -284,7 +284,7 @@ bool gl_blur_impl(double opacity, struct gl_blur_context *bctx, void *mask,
 			}
 
 			glBindTexture(GL_TEXTURE_2D, bctx->blur_textures[i]);
-			glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, tex_size->width,
+			glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA16, tex_size->width,
 			             tex_size->height, 0, GL_BGRA, GL_UNSIGNED_BYTE, NULL);
 
 			if (bctx->method == BLUR_METHOD_DUAL_KAWASE) {

src/backend/gl/gl_common.c

@@ -627,7 +627,7 @@ void gl_resize(struct gl_data *gd, int width, int height) {
 	assert(viewport_dimensions[1] >= gd->height);
 
 	glBindTexture(GL_TEXTURE_2D, gd->back_texture);
-	glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, width, height, 0, GL_BGR,
+	glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB16, width, height, 0, GL_BGR,
 	             GL_UNSIGNED_BYTE, NULL);
 
 	gl_check_err();
@@ -873,7 +873,9 @@ bool gl_init(struct gl_data *gd, session_t *ps) {
 	glUniformMatrix4fv(pml, 1, false, projection_matrix[0]);
 	glUseProgram(0);
 
-	gd->present_prog = gl_create_program_from_str(present_vertex_shader, dummy_frag);
+	gd->present_prog =
+	    gl_create_program_from_strv((const char *[]){present_vertex_shader, NULL},
+	                                (const char *[]){present_frag, dither_glsl, NULL});
 	if (!gd->present_prog) {
 		log_error("Failed to create the present shader");
 		return false;

src/backend/gl/gl_common.h

@@ -288,5 +288,6 @@ static const GLuint vert_in_texcoord_loc = 1;
 #define QUOTE(...) #__VA_ARGS__
 
 extern const char vertex_shader[], copy_with_mask_frag[], masking_glsl[], dummy_frag[],
-    fill_frag[], fill_vert[], interpolating_frag[], interpolating_vert[], win_shader_glsl[],
-    win_shader_default[], present_vertex_shader[], shadow_colorization_frag[];
+    present_frag[], fill_frag[], fill_vert[], interpolating_frag[], interpolating_vert[],
+    win_shader_glsl[], win_shader_default[], present_vertex_shader[], dither_glsl[],
+    shadow_colorization_frag[];

src/backend/gl/shaders.c

@@ -9,6 +9,15 @@ const char dummy_frag[] = GLSL(330,
 	}
 );
 
+const char present_frag[] = GLSL(330,
+	uniform sampler2D tex;
+	in vec2 texcoord;
+	vec4 dither(vec4, vec2);
+	void main() {
+		gl_FragColor = dither(texelFetch(tex, ivec2(texcoord.xy), 0), texcoord);
+	}
+);
+
 const char copy_with_mask_frag[] = GLSL(330,
 	uniform sampler2D tex;
 	in vec2 texcoord;
@@ -174,6 +183,27 @@ const char vertex_shader[] = GLSL(330,
 		texcoord = in_texcoord + texorig;
 	}
 );
+const char dither_glsl[] = GLSL(330,
+	// Stolen from: https://www.shadertoy.com/view/7sfXDn
+	float bayer2(vec2 a) {
+		a = floor(a);
+		return fract(a.x / 2. + a.y * a.y * .75);
+	}
+	// 16 * 16 is 2^8, so in total we have equivalent of 16-bit
+	// color depth, should be enough?
+	float bayer(vec2 a16) {
+		vec2  a8 = a16 * .5;
+		vec2  a4 =  a8 * .5;
+		vec2  a2 =  a4 * .5;
+		float bayer32 = ((bayer2(a2) * .25 + bayer2( a4))
+		                             * .25 + bayer2( a8))
+		                             * .25 + bayer2(a16);
+		return bayer32;
+	}
+	vec4 dither(vec4 c, vec2 coord) {
+		return vec4(c + bayer(coord) / 255.0);
+	}
+);
 const char shadow_colorization_frag[] = GLSL(330,
 	uniform vec4 color;
 	uniform sampler2D tex;