Skip to content

Use premultiplied alpha everywhere, take 2 #515

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 28, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 5 additions & 13 deletions src/BitmapSkin.js
Original file line number Diff line number Diff line change
@@ -18,9 +18,6 @@ class BitmapSkin extends Skin {
/** @type {!RenderWebGL} */
this._renderer = renderer;

/** @type {WebGLTexture} */
this._texture = null;

/** @type {Array<int>} */
this._textureSize = [0, 0];
}
@@ -95,22 +92,17 @@ class BitmapSkin extends Skin {
textureData = context.getImageData(0, 0, bitmapData.width, bitmapData.height);
}

if (this._texture) {
gl.bindTexture(gl.TEXTURE_2D, this._texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, textureData);
this._silhouette.update(textureData);
} else {
// TODO: mipmaps?
if (this._texture === null) {
const textureOptions = {
auto: true,
wrap: gl.CLAMP_TO_EDGE,
src: textureData
auto: false,
wrap: gl.CLAMP_TO_EDGE
};

this._texture = twgl.createTexture(gl, textureOptions);
this._silhouette.update(textureData);
}

this._setTexture(textureData);

// Do these last in case any of the above throws an exception
this._costumeResolution = costumeResolution || 2;
this._textureSize = BitmapSkin._getBitmapSize(bitmapData);
3 changes: 3 additions & 0 deletions src/Drawable.js
Original file line number Diff line number Diff line change
@@ -685,6 +685,9 @@ class Drawable {
const localPosition = getLocalPosition(drawable, vec);
if (localPosition[0] < 0 || localPosition[1] < 0 ||
localPosition[0] > 1 || localPosition[1] > 1) {
dst[0] = 0;
dst[1] = 0;
dst[2] = 0;
dst[3] = 0;
return dst;
}
30 changes: 25 additions & 5 deletions src/EffectTransform.js
Original file line number Diff line number Diff line change
@@ -130,15 +130,21 @@ class EffectTransform {
const effects = drawable.enabledEffects;
const uniforms = drawable.getUniforms();

if ((effects & ShaderManager.EFFECT_INFO.ghost.mask) !== 0) {
// gl_FragColor.a *= u_ghost
inOutColor[3] *= uniforms.u_ghost;
}

const enableColor = (effects & ShaderManager.EFFECT_INFO.color.mask) !== 0;
const enableBrightness = (effects & ShaderManager.EFFECT_INFO.brightness.mask) !== 0;

if (enableColor || enableBrightness) {
// gl_FragColor.rgb /= gl_FragColor.a + epsilon;
// Here, we're dividing by the (previously pre-multiplied) alpha to ensure HSV is properly calculated
// for partially transparent pixels.
// epsilon is present in the shader because dividing by 0 (fully transparent pixels) messes up calculations.
// We're doing this with a Uint8ClampedArray here, so dividing by 0 just gives 255. We're later multiplying
// by 0 again, so it won't affect results.
const alpha = inOutColor[3] / 255;
inOutColor[0] /= alpha;
inOutColor[1] /= alpha;
inOutColor[2] /= alpha;

// vec3 hsl = convertRGB2HSL(gl_FragColor.xyz);
const hsl = rgbToHsl(inOutColor);

@@ -171,6 +177,20 @@ class EffectTransform {
}
// gl_FragColor.rgb = convertHSL2RGB(hsl);
inOutColor.set(hslToRgb(hsl));

// gl_FragColor.rgb *= gl_FragColor.a + epsilon;
// Now we're doing the reverse, premultiplying by the alpha once again.
inOutColor[0] *= alpha;
inOutColor[1] *= alpha;
inOutColor[2] *= alpha;
}

if ((effects & ShaderManager.EFFECT_INFO.ghost.mask) !== 0) {
// gl_FragColor *= u_ghost
inOutColor[0] *= uniforms.u_ghost;
inOutColor[1] *= uniforms.u_ghost;
inOutColor[2] *= uniforms.u_ghost;
inOutColor[3] *= uniforms.u_ghost;
}

return inOutColor;
44 changes: 17 additions & 27 deletions src/PenSkin.js
Original file line number Diff line number Diff line change
@@ -25,6 +25,12 @@ const DefaultPenAttributes = {
diameter: 1
};

/**
* Reused memory location for storing a premultiplied pen color.
* @type {FloatArray}
*/
const __premultipliedColor = [0, 0, 0, 0];


/**
* Reused memory location for projection matrices.
@@ -88,9 +94,6 @@ class PenSkin extends Skin {
/** @type {HTMLCanvasElement} */
this._canvas = document.createElement('canvas');

/** @type {WebGLTexture} */
this._texture = null;

/** @type {WebGLTexture} */
this._exportTexture = null;

@@ -123,7 +126,7 @@ class PenSkin extends Skin {

const NO_EFFECTS = 0;
/** @type {twgl.ProgramInfo} */
this._stampShader = this._renderer._shaderManager.getShader(ShaderManager.DRAW_MODE.stamp, NO_EFFECTS);
this._stampShader = this._renderer._shaderManager.getShader(ShaderManager.DRAW_MODE.default, NO_EFFECTS);

/** @type {twgl.ProgramInfo} */
this._lineShader = this._renderer._shaderManager.getShader(ShaderManager.DRAW_MODE.lineSample, NO_EFFECTS);
@@ -154,13 +157,6 @@ class PenSkin extends Skin {
return true;
}

/**
* @returns {boolean} true if alpha is premultiplied, false otherwise
*/
get hasPremultipliedAlpha () {
return true;
}

/**
* @return {Array<number>} the "native" size, in texels, of this skin. [width, height]
*/
@@ -188,7 +184,7 @@ class PenSkin extends Skin {
clear () {
const gl = this._renderer.gl;
twgl.bindFramebufferInfo(gl, this._framebuffer);

/* Reset framebuffer to transparent black */
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT);
@@ -317,13 +313,6 @@ class PenSkin extends Skin {

twgl.bindFramebufferInfo(gl, this._framebuffer);

// Needs a blend function that blends a destination that starts with
// no alpha.
gl.blendFuncSeparate(
gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA,
gl.ONE, gl.ONE_MINUS_SRC_ALPHA
);

gl.viewport(0, 0, bounds.width, bounds.height);

gl.useProgram(currentShader.program);
@@ -344,8 +333,6 @@ class PenSkin extends Skin {
_exitDrawLineOnBuffer () {
const gl = this._renderer.gl;

gl.blendFuncSeparate(gl.ONE, gl.ONE_MINUS_SRC_ALPHA, gl.ZERO, gl.ONE);

twgl.bindFramebufferInfo(gl, null);
}

@@ -384,6 +371,13 @@ class PenSkin extends Skin {
const radius = diameter / 2;
const yScalar = (0.50001 - (radius / (length + diameter)));

// Premultiply pen color by pen transparency
const penColor = penAttributes.color4f || DefaultPenAttributes.color4f;
__premultipliedColor[0] = penColor[0] * penColor[3];
__premultipliedColor[1] = penColor[1] * penColor[3];
__premultipliedColor[2] = penColor[2] * penColor[3];
__premultipliedColor[3] = penColor[3];

const uniforms = {
u_positionScalar: yScalar,
u_capScale: diameter,
@@ -397,7 +391,7 @@ class PenSkin extends Skin {
twgl.m4.scaling(scalingVector, __modelScalingMatrix),
__modelMatrix
),
u_lineColor: penAttributes.color4f || DefaultPenAttributes.color4f
u_lineColor: __premultipliedColor
};

twgl.setUniforms(currentShader, uniforms);
@@ -490,8 +484,6 @@ class PenSkin extends Skin {

twgl.bindFramebufferInfo(gl, this._framebuffer);

gl.blendFuncSeparate(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA, gl.ONE, gl.ONE_MINUS_SRC_ALPHA);

this._drawRectangleRegionEnter(this._stampShader, this._bounds);
}

@@ -501,8 +493,6 @@ class PenSkin extends Skin {
_exitDrawToBuffer () {
const gl = this._renderer.gl;

gl.blendFuncSeparate(gl.ONE, gl.ONE_MINUS_SRC_ALPHA, gl.ZERO, gl.ONE);

twgl.bindFramebufferInfo(gl, null);
}

@@ -661,7 +651,7 @@ class PenSkin extends Skin {
skinImageData.data.set(skinPixels);
skinContext.putImageData(skinImageData, 0, 0);

this._silhouette.update(this._canvas);
this._silhouette.update(this._canvas, true /* isPremultiplied */);

this._silhouetteDirty = false;
}
27 changes: 9 additions & 18 deletions src/RenderWebGL.js
Original file line number Diff line number Diff line change
@@ -196,7 +196,7 @@ class RenderWebGL extends EventEmitter {
gl.disable(gl.DEPTH_TEST);
/** @todo disable when no partial transparency? */
gl.enable(gl.BLEND);
gl.blendFuncSeparate(gl.ONE, gl.ONE_MINUS_SRC_ALPHA, gl.ZERO, gl.ONE);
gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
}

/**
@@ -834,7 +834,8 @@ class RenderWebGL extends EventEmitter {
projection,
{
extraUniforms,
ignoreVisibility: true // Touching color ignores sprite visibility
ignoreVisibility: true, // Touching color ignores sprite visibility,
effectMask: ~ShaderManager.EFFECT_INFO.ghost.mask
});

gl.stencilFunc(gl.EQUAL, 1, 1);
@@ -1554,7 +1555,7 @@ class RenderWebGL extends EventEmitter {
const projection = twgl.m4.ortho(bounds.left, bounds.right, bounds.top, bounds.bottom, -1, 1);

// Draw the stamped sprite onto the PenSkin's framebuffer.
this._drawThese([stampID], ShaderManager.DRAW_MODE.stamp, projection, {ignoreVisibility: true});
this._drawThese([stampID], ShaderManager.DRAW_MODE.default, projection, {ignoreVisibility: true});
skin._silhouetteDirty = true;
}

@@ -1744,14 +1745,6 @@ class RenderWebGL extends EventEmitter {
}

twgl.setUniforms(currentShader, uniforms);

/* adjust blend function for this skin */
if (drawable.skin.hasPremultipliedAlpha){
gl.blendFuncSeparate(gl.ONE, gl.ONE_MINUS_SRC_ALPHA, gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
} else {
gl.blendFuncSeparate(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA, gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
}

twgl.drawBufferInfo(gl, this._bufferInfo, gl.TRIANGLES);
}

@@ -1902,13 +1895,11 @@ class RenderWebGL extends EventEmitter {
}
*/
Drawable.sampleColor4b(vec, drawables[index].drawable, __blendColor);
// if we are fully transparent, go to the next one "down"
const sampleAlpha = __blendColor[3] / 255;
// premultiply alpha
dst[0] += __blendColor[0] * blendAlpha * sampleAlpha;
dst[1] += __blendColor[1] * blendAlpha * sampleAlpha;
dst[2] += __blendColor[2] * blendAlpha * sampleAlpha;
blendAlpha *= (1 - sampleAlpha);
// Equivalent to gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA)
dst[0] += __blendColor[0] * blendAlpha;
dst[1] += __blendColor[1] * blendAlpha;
dst[2] += __blendColor[2] * blendAlpha;
blendAlpha *= (1 - (__blendColor[3] / 255));
}
// Backdrop could be transparent, so we need to go to the "clear color" of the
// draw scene (white) as a fallback if everything was alpha
3 changes: 2 additions & 1 deletion src/SVGSkin.js
Original file line number Diff line number Diff line change
@@ -90,7 +90,8 @@ class SVGSkin extends Skin {
const textureOptions = {
auto: false,
wrap: this._renderer.gl.CLAMP_TO_EDGE,
src: textureData
src: textureData,
premultiplyAlpha: true
};

const mip = twgl.createTexture(this._renderer.gl, textureOptions);
7 changes: 1 addition & 6 deletions src/ShaderManager.js
Original file line number Diff line number Diff line change
@@ -171,12 +171,7 @@ ShaderManager.DRAW_MODE = {
/**
* Sample a "texture" to draw a line with caps.
*/
lineSample: 'lineSample',

/**
* Draw normally except for pre-multiplied alpha
*/
stamp: 'stamp'
lineSample: 'lineSample'
};

module.exports = ShaderManager;
55 changes: 46 additions & 9 deletions src/Silhouette.js
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@ let __SilhouetteUpdateCanvas;
* @return {number} Alpha value for x/y position
*/
const getPoint = ({_width: width, _height: height, _colorData: data}, x, y) => {
// 0 if outside bouds, otherwise read from data.
// 0 if outside bounds, otherwise read from data.
if (x >= width || y >= height || x < 0 || y < 0) {
return 0;
}
@@ -39,14 +39,39 @@ const __cornerWork = [

/**
* Get the color from a given silhouette at an x/y local texture position.
* Multiply color values by alpha for proper blending.
* @param {Silhouette} The silhouette to sample.
* @param {number} x X position of texture (0-1).
* @param {number} y Y position of texture (0-1).
* @param {Uint8ClampedArray} dst A color 4b space.
* @return {Uint8ClampedArray} The dst vector.
*/
const getColor4b = ({_width: width, _height: height, _colorData: data}, x, y, dst) => {
// 0 if outside bouds, otherwise read from data.
// 0 if outside bounds, otherwise read from data.
if (x >= width || y >= height || x < 0 || y < 0) {
return dst.fill(0);
}
const offset = ((y * width) + x) * 4;
// premultiply alpha
const alpha = data[offset + 3] / 255;
dst[0] = data[offset] * alpha;
dst[1] = data[offset + 1] * alpha;
dst[2] = data[offset + 2] * alpha;
dst[3] = data[offset + 3];
return dst;
};

/**
* Get the color from a given silhouette at an x/y local texture position.
* Do not multiply color values by alpha, as it has already been done.
* @param {Silhouette} The silhouette to sample.
* @param {number} x X position of texture (0-1).
* @param {number} y Y position of texture (0-1).
* @param {Uint8ClampedArray} dst A color 4b space.
* @return {Uint8ClampedArray} The dst vector.
*/
const getPremultipliedColor4b = ({_width: width, _height: height, _colorData: data}, x, y, dst) => {
// 0 if outside bounds, otherwise read from data.
if (x >= width || y >= height || x < 0 || y < 0) {
return dst.fill(0);
}
@@ -78,15 +103,21 @@ class Silhouette {
*/
this._colorData = null;

// By default, silhouettes are assumed not to contain premultiplied image data,
// so when we get a color, we want to multiply it by its alpha channel.
// Point `_getColor` to the version of the function that multiplies.
this._getColor = getColor4b;

this.colorAtNearest = this.colorAtLinear = (_, dst) => dst.fill(0);
}

/**
* Update this silhouette with the bitmapData for a skin.
* @param {*} bitmapData An image, canvas or other element that the skin
* @param {ImageData|HTMLCanvasElement|HTMLImageElement} bitmapData An image, canvas or other element that the skin
* @param {boolean} isPremultiplied True if the source bitmap data comes premultiplied (e.g. from readPixels).
* rendering can be queried from.
*/
update (bitmapData) {
update (bitmapData, isPremultiplied = false) {
let imageData;
if (bitmapData instanceof ImageData) {
// If handed ImageData directly, use it directly.
@@ -109,6 +140,12 @@ class Silhouette {
imageData = ctx.getImageData(0, 0, width, height);
}

if (isPremultiplied) {
this._getColor = getPremultipliedColor4b;
} else {
this._getColor = getColor4b;
}

this._colorData = imageData.data;
// delete our custom overriden "uninitalized" color functions
// let the prototype work for itself
@@ -124,7 +161,7 @@ class Silhouette {
* @returns {Uint8ClampedArray} dst
*/
colorAtNearest (vec, dst) {
return getColor4b(
return this._getColor(
this,
Math.floor(vec[0] * (this._width - 1)),
Math.floor(vec[1] * (this._height - 1)),
@@ -151,10 +188,10 @@ class Silhouette {
const xFloor = Math.floor(x);
const yFloor = Math.floor(y);

const x0y0 = getColor4b(this, xFloor, yFloor, __cornerWork[0]);
const x1y0 = getColor4b(this, xFloor + 1, yFloor, __cornerWork[1]);
const x0y1 = getColor4b(this, xFloor, yFloor + 1, __cornerWork[2]);
const x1y1 = getColor4b(this, xFloor + 1, yFloor + 1, __cornerWork[3]);
const x0y0 = this._getColor(this, xFloor, yFloor, __cornerWork[0]);
const x1y0 = this._getColor(this, xFloor + 1, yFloor, __cornerWork[1]);
const x0y1 = this._getColor(this, xFloor, yFloor + 1, __cornerWork[2]);
const x1y1 = this._getColor(this, xFloor + 1, yFloor + 1, __cornerWork[3]);

dst[0] = (x0y0[0] * x0D * y0D) + (x0y1[0] * x0D * y1D) + (x1y0[0] * x1D * y0D) + (x1y1[0] * x1D * y1D);
dst[1] = (x0y0[1] * x0D * y0D) + (x0y1[1] * x0D * y1D) + (x1y0[1] * x1D * y0D) + (x1y1[1] * x1D * y1D);
27 changes: 20 additions & 7 deletions src/Skin.js
Original file line number Diff line number Diff line change
@@ -33,6 +33,9 @@ class Skin extends EventEmitter {
/** @type {Vec3} */
this._rotationCenter = twgl.v3.create(0, 0);

/** @type {WebGLTexture} */
this._texture = null;

/**
* The uniforms to be used by the vertex and pixel shaders.
* Some of these are used by other parts of the renderer as well.
@@ -76,13 +79,6 @@ class Skin extends EventEmitter {
return false;
}

/**
* @returns {boolean} true if alpha is premultiplied, false otherwise
*/
get hasPremultipliedAlpha () {
return false;
}

/**
* @return {int} the unique ID for this Skin.
*/
@@ -171,6 +167,23 @@ class Skin extends EventEmitter {
*/
updateSilhouette () {}

/**
* Set this skin's texture to the given image.
* @param {ImageData|HTMLCanvasElement} textureData - The canvas or image data to set the texture to.
*/
_setTexture (textureData) {
const gl = this._renderer.gl;

gl.bindTexture(gl.TEXTURE_2D, this._texture);
// Premultiplied alpha is necessary for proper blending.
// See http://www.realtimerendering.com/blog/gpus-prefer-premultiplication/
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, textureData);
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false);

this._silhouette.update(textureData);
}

/**
* Set the contents of this skin to an empty skin.
* @fires Skin.event:WasAltered
7 changes: 1 addition & 6 deletions src/TextBubbleSkin.js
Original file line number Diff line number Diff line change
@@ -42,9 +42,6 @@ class TextBubbleSkin extends Skin {
/** @type {HTMLCanvasElement} */
this._canvas = document.createElement('canvas');

/** @type {WebGLTexture} */
this._texture = null;

/** @type {Array<number>} */
this._size = [0, 0];

@@ -272,9 +269,7 @@ class TextBubbleSkin extends Skin {
this._texture = twgl.createTexture(gl, textureOptions);
}

gl.bindTexture(gl.TEXTURE_2D, this._texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, textureData);
this._silhouette.update(textureData);
this._setTexture(textureData);
}

return this._texture;
47 changes: 31 additions & 16 deletions src/shaders/sprite.frag
Original file line number Diff line number Diff line change
@@ -43,14 +43,16 @@ uniform sampler2D u_skin;

varying vec2 v_texCoord;

// Add this to divisors to prevent division by 0, which results in NaNs propagating through calculations.
// Smaller values can cause problems on some mobile devices.
const float epsilon = 1e-3;

#if !defined(DRAW_MODE_silhouette) && (defined(ENABLE_color))
// Branchless color conversions based on code from:
// http://www.chilliant.com/rgb2hsv.html by Ian Taylor
// Based in part on work by Sam Hocevar and Emil Persson
// See also: https://en.wikipedia.org/wiki/HSL_and_HSV#Formal_derivation

// Smaller values can cause problems on some mobile devices
const float epsilon = 1e-3;

// Convert an RGB color to Hue, Saturation, and Value.
// All components of input and output are expected to be in the [0,1] range.
@@ -153,16 +155,12 @@ void main()

gl_FragColor = texture2D(u_skin, texcoord0);

#ifdef ENABLE_ghost
gl_FragColor.a *= u_ghost;
#endif // ENABLE_ghost
#if defined(ENABLE_color) || defined(ENABLE_brightness)
// Divide premultiplied alpha values for proper color processing
// Add epsilon to avoid dividing by 0 for fully transparent pixels
gl_FragColor.rgb = clamp(gl_FragColor.rgb / (gl_FragColor.a + epsilon), 0.0, 1.0);

#ifdef DRAW_MODE_silhouette
// switch to u_silhouetteColor only AFTER the alpha test
gl_FragColor = u_silhouetteColor;
#else // DRAW_MODE_silhouette

#if defined(ENABLE_color)
#ifdef ENABLE_color
{
vec3 hsv = convertRGB2HSV(gl_FragColor.xyz);

@@ -178,11 +176,29 @@ void main()

gl_FragColor.rgb = convertHSV2RGB(hsv);
}
#endif // defined(ENABLE_color)
#endif // ENABLE_color

#if defined(ENABLE_brightness)
#ifdef ENABLE_brightness
gl_FragColor.rgb = clamp(gl_FragColor.rgb + vec3(u_brightness), vec3(0), vec3(1));
#endif // defined(ENABLE_brightness)
#endif // ENABLE_brightness

// Re-multiply color values
gl_FragColor.rgb *= gl_FragColor.a + epsilon;

#endif // defined(ENABLE_color) || defined(ENABLE_brightness)

#ifdef ENABLE_ghost
gl_FragColor *= u_ghost;
#endif // ENABLE_ghost

#ifdef DRAW_MODE_silhouette
// Discard fully transparent pixels for stencil test
if (gl_FragColor.a == 0.0) {
discard;
}
// switch to u_silhouetteColor only AFTER the alpha test
gl_FragColor = u_silhouetteColor;
#else // DRAW_MODE_silhouette

#ifdef DRAW_MODE_colorMask
vec3 maskDistance = abs(gl_FragColor.rgb - u_colorMask);
@@ -195,8 +211,7 @@ void main()
#endif // DRAW_MODE_silhouette

#else // DRAW_MODE_lineSample
gl_FragColor = u_lineColor;
gl_FragColor.a *= clamp(
gl_FragColor = u_lineColor * clamp(
// Scale the capScale a little to have an aliased region.
(u_capScale + u_aliasAmount -
u_capScale * 2.0 * distance(v_texCoord, vec2(0.5, 0.5))