linkImage and Video Filters

The process of applying a filter to an image or video can be easily done by using the shaders functionality of P5js, so we can transform videos and images in an optimal way using hardware.

linkInverse Color

In order to invert the color of an image or video using shaders we create a fragment shader that takes the image or video as a texture and change its rgb value with the formula rgb = 1 - rgb, in addition it is necessary to invert the y orientation of the texture, because the shader always receives it upside down. The vertex shader used is the general vertex shader described on "thebookofshaders" which just adjust the position of the texture. After the texture is processed it is applied to a rectangle in the scene as a texture.

inverse_color_shader.js1linklet inverseShader;

2linklet shaderTexture;

3link

4link

5linkfunction preload() {

6link  img = loadImage('/vc/docs/sketches/lenna.png');

7link  video = createVideo(['/vc/docs/sketches/fingers.mov', '/vc/docs/sketches/fingers.webm']);

8link  inverseShader = loadShader('/vc/docs/sketches/inverse.vert', '/vc/docs/sketches/inverse.frag');

9link  video.hide();

10link}

11link

12linkfunction setup() {

13link  createCanvas(512, 256, WEBGL);

14link  shaderTexture = createGraphics(256, 256, WEBGL);

15link  shaderTexture.noStroke();

16link

17link  video.loop();

18link  noStroke();

19link}

20link

21linkfunction draw() {

22link  shaderTexture.shader(inverseShader);

23link  inverseShader.setUniform('tex', img);

24link  texture(shaderTexture);

25link  shaderTexture.rect(0,0,256,256);

26link  rect(-256,-256/2.0,256,256)

27link  inverseShader.setUniform('tex', video);

28link  texture(shaderTexture);

29link  shaderTexture.rect(0,0,256,256);

30link  rect(0,-256/2.0,256,256)

31link}

inverse.vert1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkattribute vec3 aPosition;

6linkattribute vec2 aTexCoord;

7link

8linkvarying vec2 vTexCoord;

9link

10linkvoid main() {

11link  vTexCoord = aTexCoord;

12link

13link  vec4 positionVec4 = vec4(aPosition, 1.0);

14link  positionVec4.xy = positionVec4.xy * 2.0 - 1.0;

15link

16link  gl_Position = positionVec4;

17link}

inverse.frag1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkvarying vec2 vTexCoord;

6link

7linkuniform sampler2D tex;

8link

9link

10linkvoid main() {

11link  vec2 uv = vTexCoord;

12link  uv.y = 1.0 - uv.y;

13link

14link  vec4 tex_f = texture2D(tex, uv);

15link

16link  tex_f.rgb = 1.0 - tex_f.rgb;

17link

18link  gl_FragColor = tex_f;

19link}

linkGray Scale Luma

To apply the gray scale Luma on an image or video using shaders we apply the same technique of the inverse color, with the difference that we create a function called luma that transform the color of the texture using the dot product between the rgb vector and the vector (0.299, 0.587, 0.114) and then replace the color of the texture by the result of that dot product.

gray_color_shader.js1linklet grayShader;

2linklet shaderTexture;

3link

4linkfunction preload() {

5link  img = loadImage('/vc/docs/sketches/lenna.png');

6link  video = createVideo(['/vc/docs/sketches/fingers.mov', '/vc/docs/sketches/fingers.webm']);

7link  grayShader = loadShader('/vc/docs/sketches/gray.vert', '/vc/docs/sketches/gray.frag');

8link  video.hide();

9link}

10link

11linkfunction setup() {

12link  createCanvas(512, 256, WEBGL);

13link  shaderTexture = createGraphics(256, 256, WEBGL);

14link  shaderTexture.noStroke();

15link

16link video.loop();

17link  noStroke();

18link}

19link

20linkfunction draw() {

21link  shaderTexture.shader(grayShader);

22link  grayShader.setUniform('tex', img);

23link  texture(shaderTexture);

24link  shaderTexture.rect(0,0,256,256);

25link  rect(-256,-256/2.0,256,256)

26link  grayShader.setUniform('tex', video);

27link  texture(shaderTexture);

28link  shaderTexture.rect(0,0,256,256);

29link  rect(0,-256/2.0,256,256)

30link}

31link

gray.vert1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkattribute vec3 aPosition;

6linkattribute vec2 aTexCoord;

7link

8linkvarying vec2 vTexCoord;

9link

10linkvoid main() {

11link  vTexCoord = aTexCoord;

12link

13link  vec4 positionVec4 = vec4(aPosition, 1.0);

14link  positionVec4.xy = positionVec4.xy * 2.0 - 1.0;

15link

16link  gl_Position = positionVec4;

17link}

gray.frag1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkvarying vec2 vTexCoord;

6link

7linkuniform sampler2D tex;

8link

9linkfloat luma(vec3 color) {

10link  return dot(color, vec3(0.299, 0.587, 0.114));

11link}

12link

13linkvoid main() {

14link  vec2 uv = vTexCoord;

15link  uv.y = 1.0 - uv.y;

16link

17link  vec4 tex_f = texture2D(tex, uv);

18link

19link  float gray = luma(tex_f.rgb);

20link

21link  gl_FragColor = vec4(gray,gray,gray,1.0);

22link}

linkGray Scale RGB

In order to apply the gray scale RGB on an image or video using shaders we apply the same technique of the inverse color, with the difference that we create a function called rgb that transform the color of the texture using the dot product between the rgb vector and the vector (1/3.0, 1/3.0, 1/3.0) and then replace the color of the texture by the result of that dot product.

gray_color_shader.js1linklet grayShader;

2linklet shaderTexture;

3link

4linkfunction preload() {

5link  img = loadImage('/vc/docs/sketches/lenna.png');

6link  video = createVideo(['/vc/docs/sketches/fingers.mov', '/vc/docs/sketches/fingers.webm']);

7link  grayShader = loadShader('/vc/docs/sketches/gray_rgb.vert', '/vc/docs/sketches/gray_rgb.frag');

8link  video.hide();

9link}

10link

11linkfunction setup() {

12link  createCanvas(512, 256, WEBGL);

13link  shaderTexture = createGraphics(256, 256, WEBGL);

14link  shaderTexture.noStroke();

15link

16link video.loop();

17link  noStroke();

18link}

19link

20linkfunction draw() {

21link  shaderTexture.shader(grayShader);

22link  grayShader.setUniform('tex', img);

23link  texture(shaderTexture);

24link  shaderTexture.rect(0,0,256,256);

25link  rect(-256,-256/2.0,256,256)

26link  grayShader.setUniform('tex', video);

27link  texture(shaderTexture);

28link  shaderTexture.rect(0,0,256,256);

29link  rect(0,-256/2.0,256,256)

30link}

31link

gray_rgb.vert1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkattribute vec3 aPosition;

6linkattribute vec2 aTexCoord;

7link

8linkvarying vec2 vTexCoord;

9link

10linkvoid main() {

11link  vTexCoord = aTexCoord;

12link

13link  vec4 positionVec4 = vec4(aPosition, 1.0);

14link  positionVec4.xy = positionVec4.xy * 2.0 - 1.0;

15link

16link  gl_Position = positionVec4;

17link}

gray_rgb.frag1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkvarying vec2 vTexCoord;

6link

7linkuniform sampler2D tex;

8link

9linkfloat rgb(vec3 color) {

10link  return dot(color, vec3(1.0/3.0, 1.0/3.0, 1.0/3.0));

11link}

12link

13linkvoid main() {

14link  vec2 uv = vTexCoord;

15link  uv.y = 1.0 - uv.y;

16link

17link  vec4 tex_f = texture2D(tex, uv);

18link

19link  float gray = rgb(tex_f.rgb);

20link

21link  gl_FragColor = vec4(gray,gray,gray,1.0);

22link}

linkTry it with your own camera

Now you can try these filters with your own camera.

linkComparison with Software implementation

When we measure the framerate of both implementations using the same images as the base dataset we found an improvement 196% in the frameRate, on the software implementation the value of the frameRate was around 8.23 frames per second, while in the hardware implementation we have 24.39 frames per second on average.

Image and Video Filters Inverse Color Gray Scale Luma Gray Scale RGB Try it with your own camera Comparison with Software implementation