vc | Kernel (image processing)

linkKernel (image processing)

Kernel methods have been seen on the software image & video processing, but this kind of methods can be implemented using the multiplication between the image and the kernel on shaders. The idea of the implementation is to take the kernel matrix, locate it on the center of the pixel that we want to transform and apply products element by element between the adjacent pixels of the images and the kernel. The implementation of multiple kernels will be seen in detail on the following pages, but by now you can test your own filters on this section.

kernel.js1linklet capture;

2linklet button;

3linklet grayBtn;

4linklet invertBtn;

5linklet normalBtn;

6linklet kernelShader;

7linklet shaderTexture;

8linklet offset = [];

9linklet inputs = [];

10linklet kernel = [];

11linklet size;

12linklet borderButton;

13linklet focusButton;

14linklet blurryButton;

15linklet blurry5Button;

16linklet normalize = false;

17linkfunction setup() {

18link  shaderTexture = createGraphics(640, 480, WEBGL);

19link  shaderTexture.noStroke();

20link  createCanvas(1000, 480, WEBGL);

21link  background(0,0,0);

22link

23link  kernelShader = loadShader('/vc/docs/sketches/kernel.vert', '/vc/docs/sketches/kernel.frag');

24link  input = createInput();

25link  input.style("padding", "8px");

26link  input.style("display", "block");

27link  input.style("border", "none");

28link  input.style("border-bottom", "1px solid #ccc");

29link  input.style("font-family","'Roboto',sans-serif");

30link  input.style("font-weight","300");

31link  borderButton = createButton('Border Filter');

32link  focusButton = createButton('Focus Filter');

33link  blurryButton = createButton('Gaussian Blurry Filter');

34link  blurry5Button = createButton('Blurry 5x5 Filter');

35link  borderButton.position(670, 300);

36link  borderButton.mousePressed(() => setMatrix([-1, -1, -1, -1, 8, -1, -1, -1, -1], 3, false))

37link  focusButton.position(670, 350);

38link  focusButton.mousePressed(() => setMatrix([0, -1, 0, -1, 5, -1, 0, -1, 0], 3, true))

39link

40link  blurryButton.position(670, 400);

41link  blurryButton.mousePressed(() => setMatrix([1/16, 2/16, 1/16, 2/16, 4/16, 2/16, 1/16, 2/16, 1/16], 3, true))

42link

43link  blurry5Button.position(670, 450);

44link  blurry5Button.mousePressed(() => setMatrix([1/256, 4/256, 6/256, 4/256, 1/256, 4/256, 16/256, 24/256, 16/256, 4/256, 

45link    6/256, 24/256, 36/256, 24/256, 6/256, 4/256, 16/256, 24/256, 16/256, 4/256, 1/256, 4/256, 6/256, 4/256, 1/256], 5, true))

46link

47link  button2 = createButton('submit');

48link  button2.position(850, 53);

49link  button2.mousePressed(convMatrix);

50link  button2.style("display","inline-block");

51link  button2.style("padding","0.35em 1.2em");

52link  button2.style("border","0.1em solid #FFFFFF");

53link  button2.style("margin","0 0.3em 0.3em 0");

54link  button2.style("border-radius","0.12em");

55link  button2.style("box-sizing","border-box");

56link  button2.style("text-decoration","none");

57link  button2.style("font-family","'Roboto',sans-serif");

58link  button2.style("font-weight","300");

59link  button2.style("color","#FFFFFF");

60link  button2.style("text-align","center");

61link  button2.style("background","transparent");

62link  input.position(670, 50);

63link  head = createElement('h3', 'Length and Width of the matrix');

64link  head.position(670, 0);

65link  head.style("font-family","'Roboto',sans-serif");

66link  head.style("font-weight","300");

67link  head.style("color","#FFFFFF");

68link

69link  [borderButton, focusButton, blurryButton, blurry5Button].forEach(button2 => {

70link    button2.style("display","inline-block");

71link  button2.style("padding","0.35em 1.2em");

72link  button2.style("border","0.1em solid #FFFFFF");

73link  button2.style("margin","0 0.3em 0.3em 0");

74link  button2.style("border-radius","0.12em");

75link  button2.style("box-sizing","border-box");

76link  button2.style("text-decoration","none");

77link  button2.style("font-family","'Roboto',sans-serif");

78link  button2.style("font-weight","300");

79link  button2.style("color","#FFFFFF");

80link  button2.style("text-align","center");

81link  button2.style("background","transparent");

82link  })

83link}

84link

85linkfunction setMatrix(mtrx, sz, nrml) {

86link  size = sz;

87link  normalize = nrml;

88link  kernel = mtrx;

89link  offset = [];

90link  for (let i = 0; i < size; i++) {

91link    for (let j = 0; j < size; j++) {

92link      offset.push(float(-(size-1)/2.0+j)*1.0/height);

93link      offset.push(float(-(size-1)/2.0+i)*1.0/width);

94link    }

95link  }

96link

97link  for (let i = size*size; i < 49; i++) {

98link    offset.push(0);

99link    kernel.push(0);

100link  }

101link  if (!capture) {

102link    capture = createCapture(VIDEO);

103link  capture.size(640, 480);

104link  capture.hide();

105link  }

106link

107link}

108link

109linkfunction startCapture() {

110link  kernel = []

111link  offset = []

112link  normalize = false;

113link  for (let i = 0; i < size; i++) {

114link    for (let j = 0; j < size; j++) {

115link      let index = j + i*size;

116link      kernel.push(inputs[index].value());

117link    }

118link  }

119link  for (let i = 0; i < size; i++) {

120link    for (let j = 0; j < size; j++) {

121link      offset.push(float(-(size-1)/2.0+j)*1.0/height);

122link      offset.push(float(-(size-1)/2.0+i)*1.0/width);

123link    }

124link  }

125link

126link  for (let i = size*size; i < 49*2; i++) {

127link    offset.push(0);

128link    kernel.push(0);

129link  }

130link  if (!capture) {

131link    capture = createCapture(VIDEO);

132link  capture.size(640, 480);

133link  capture.hide();

134link  }

135link

136link}

137link

138link

139linkfunction draw() {

140link  if (capture) {    

141link    shaderTexture.shader(kernelShader);

142link    kernelShader.setUniform('tex0', capture);

143link    kernelShader.setUniform('kernel', kernel);

144link    kernelShader.setUniform('n', size);

145link    kernelShader.setUniform('size', 1);

146link    kernelShader.setUniform('ofs', offset)

147link    texture(shaderTexture);

148link    shaderTexture.rect(0,0,256,256);

149link    rect(-1000  /2.0,-240.0,640,480)

150link  } 

151link

152link}

153link

154link

155linkfunction convMatrix() {

156link  size = input.value();

157link  inputs.forEach(input => {

158link    input.remove();

159link  })

160link  inputs = [];

161link  for(let i = 0; i < size; i++) {

162link    for (let j = 0; j < size; j++) {

163link      let aux = createInput();

164link      aux.style("display", "block");

165link      aux.style("border", "none");

166link      aux.style("border-bottom", "1px solid #ccc");

167link      aux.style("font-family","'Roboto',sans-serif");

168link      aux.style("font-weight","300");

169link      aux.position(670 + i*30, 100 + j*20);

170link      aux.size(20);

171link      inputs.push(aux)

172link    }

173link  }

174link  let button3 = createButton('Use kernel');

175link  button3.style("display","inline-block");

176link  button3.style("padding","0.35em 1.2em");

177link  button3.style("border","0.1em solid #FFFFFF");

178link  button3.style("margin","0 0.3em 0.3em 0");

179link  button3.style("border-radius","0.12em");

180link  button3.style("box-sizing","border-box");

181link  button3.style("text-decoration","none");

182link  button3.style("font-family","'Roboto',sans-serif");

183link  button3.style("font-weight","300");

184link  button3.style("color","#FFFFFF");

185link  button3.style("text-align","center");

186link  button3.style("background","transparent");

187link  button3.position(670, 250);

188link  button3.mousePressed(startCapture);

189link}

190link

kernel.vert1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkattribute vec3 aPosition;

6linkattribute vec2 aTexCoord;

7link

8linkvarying vec2 vTexCoord;

9link

10linkvoid main() {

11link  vTexCoord = aTexCoord;

12link

13link  vec4 positionVec4 = vec4(aPosition, 1.0);

14link  positionVec4.xy = positionVec4.xy * 2.0 - 1.0;

15link

16link  gl_Position = positionVec4;

17link}

kernel.frag1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkvarying vec2 vTexCoord;

6linkuniform int n;

7linkuniform float kernel[49];

8linkuniform float size;

9linkuniform sampler2D tex0;

10linkuniform float ofs[98];

11link

12link

13linkvec4 conv = vec4(0.0);

14link

15linkvoid main(){

16link

17link    vec2 uv = vTexCoord;

18link  uv.y = 1.0 - uv.y;

19link

20link

21link    for(int i = 0; i<49; i++){

22link    if (i >= int(n*n)) break;

23link        vec4 color = texture2D(tex0, uv + vec2(ofs[i*2], ofs[i*2+1])*size);

24link

25link        conv += color*kernel[i] ;

26link  }

27link

28link    gl_FragColor = vec4(conv.rgb, 1.0);

29link}

linkComparison with Software implementation

When we measure the framerate of both implementations using an edge detection kernel we found an improvement 867% in the frameRate, on the software implementation the value of the frameRate was around 5.84 frames per second, while in the hardware implementation we have 56.49 frames per second on average. Additionally the lag on the software implementation video is completely evident, being impossible to have a fluid image, while in the hardware implementation the fluency is evident.

Software Image & Video Processingchevron_right

Problem Statement and Background Image and Video filters Image Photographic-Mosaic Ascii art

Kernel (Image processing)chevron_right

Kernel (Image processing) Edge detection Sharpen Blur Unsharp Masking Your own filters

Conclusions and References

Hardware Image & Video Processingchevron_right

Problem Statement and Background Image and Video filters Ascii art Photo Mosaic

Kernel (Image processing)chevron_right

Kernel (Image processing) Edge detection Sharpen Blur Unsharp Masking

Conclusions and References

Renderingchevron_right

Problem Statement and Background Radiosity Algorithm Rasterization using Barycentric coordinates Conclusions and References

Computer Graphics HCI