vc | Photo mosaic

linkPhoto mosaic

Photo mosaic is the technique of creating a picture or video with a dataset of small images. In order to implement it using shaders it is necessary first to create a single texture image with all the small images that we want to use, then we tile the original image depending on the number of tiles that we want to use, and we calculate the brightness of each tile. With that brightness we can select the proper subimage on the texture image and then we draw the corresponding pixel on the original image.

linkTry it with your own camera

mosaic_hardware.js1linklet camShader;

2linklet input;

3linklet button2;

4linklet cam;

5linklet tiles = 100.0;

6linklet allImages = [];

7linklet brightnessValues = [];

8linklet imgTexture;

9linkfunction transform(index) {

10link  let val = "";

11link  index = str(index);

12link  for (let i = 0; i < 4-index.length;i++) {

13link    val += "0";

14link  }

15link  val += index;

16link  return val;

17link}

18link

19linkfunction preload() {

20link  camShader = loadShader('/vc/docs/sketches/mosaic.vert', '/vc/docs/sketches/mosaic.frag');

21link  for (i = 0; i < 100; i++) {

22link    now = "/vc/docs/sketches/apmw_birds/apmw_base_birds_" + transform(i+1) + '.jpg';

23link    allImages[i] = loadImage(now);

24link  }

25link}

26link

27linkfunction setup() {

28link  shaderTexture = createGraphics(640, 480, WEBGL);

29link  shaderTexture.noStroke();

30link  createCanvas(800, windowHeight, WEBGL);

31link  background(0,0,0);

32link

33link  noStroke();

34link

35link  input = createInput(100);

36link  input.style("padding", "8px");

37link  input.style("display", "block");

38link  input.style("border", "none");

39link  input.style("border-bottom", "1px solid #ccc");

40link  input.style("font-family","'Roboto',sans-serif");

41link  input.style("font-weight","300");

42link  button2 = createButton('submit');

43link  button2.position(600, 150);

44link  button2.mousePressed(changeNumberOfTiles)

45link  button2.style("display","inline-block");

46link  button2.style("padding","0.35em 1.2em");

47link  button2.style("border","0.1em solid #FFFFFF");

48link  button2.style("margin","0 0.3em 0.3em 0");

49link  button2.style("border-radius","0.12em");

50link  button2.style("box-sizing","border-box");

51link  button2.style("text-decoration","none");

52link  button2.style("font-family","'Roboto',sans-serif");

53link  button2.style("font-weight","300");

54link  button2.style("color","#FFFFFF");

55link  button2.style("text-align","center");

56link  button2.style("background","transparent");

57link  input.position(600, 70);

58link  head = createElement('h3', 'Number of tiles per row and column');

59link  head.position(600, 0);

60link  head.style("font-family","'Roboto',sans-serif");

61link  head.style("font-weight","300");

62link  head.style("color","#FFFFFF");

63link  cam = createCapture(VIDEO);

64link  cam.size(500, 400);

65link

66link  cam.hide();

67link  imgTexture = createImage(500/tiles*100, windowHeight/tiles)

68link  for (i = 0; i < 100; i++) {

69link    now = "/vc/docs/sketches/apmw_birds/apmw_base_birds_" + transform(i+1) + '.jpg';

70link    let image = allImages[i]

71link    allImages[i] = createImage(500/tiles, windowHeight/tiles);

72link    allImages[i].copy(image, 0, 0, image.width, image.height, 0, 0, 500/tiles, windowHeight/tiles);

73link    imgTexture.copy(image, 0, 0, image.width, image.height, i*500/tiles, 0, 500/tiles, windowHeight/tiles);

74link    image.loadPixels();

75link

76link    let avg = 0;

77link    for (let j = 0; j < image.height; j++) {

78link      for (let k = 0; k < image.width; k++) {

79link        let index = (k + j*image.width);

80link        let r = image.pixels[index*4]/255.0;

81link        let g = image.pixels[index*4+1]/255.0;

82link        let b = image.pixels[index*4+2]/255.0;

83link        let gray = r *0.2126 + g *0.7152 + b *0.0722;

84link        avg += gray;

85link      }

86link    }

87link    avg /= (image.height*image.width);

88link    brightnessValues[i] = avg;

89link  }

90link}

91link

92linkfunction changeNumberOfTiles() {

93link  tiles = input.value();

94link  for (i = 0; i < 100; i++) {

95link    now = "/vc/docs/sketches/apmw_birds/apmw_base_birds_" + transform(i+1) + '.jpg';

96link    let image = allImages[i]

97link    allImages[i] = createImage(500/tiles, windowHeight/tiles);

98link    allImages[i].copy(image, 0, 0, image.width, image.height, 0, 0, 500/tiles, windowHeight/tiles);

99link    imgTexture.copy(image, 0, 0, image.width, image.height, i*500/tiles, 0, 500/tiles, windowHeight/tiles);

100link    image.loadPixels();

101link  }

102link}

103link

104linkfunction draw() {  

105link

106link    shaderTexture.shader(camShader);

107link    camShader.setUniform('tex0', cam);

108link  camShader.setUniform('textures', imgTexture);

109link  camShader.setUniform('tiles', tiles);

110link  camShader.setUniform('brightValues', brightnessValues);

111link  camShader.setUniform('width', 500.0/tiles);

112link

113link    texture(shaderTexture);

114link    shaderTexture.rect(0,0,256,256);

115link    rect(-1000  /2.0,-240.0,640,480)

116link

117link}

mosaic.vert1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkattribute vec3 aPosition;

6linkattribute vec2 aTexCoord;

7link

8linkvarying vec2 vTexCoord;

9link

10linkvoid main() {

11link  vTexCoord = aTexCoord;

12link

13link  vec4 positionVec4 = vec4(aPosition, 1.0);

14link  positionVec4.xy = positionVec4.xy * 2.0 - 1.0;

15link

16link  gl_Position = positionVec4;

17link}

mosaic.frag1link#ifdef GL_ES

2linkprecision mediump float;

3link#endif

4link

5linkvarying vec2 vTexCoord;

6link

7linkuniform sampler2D tex0;

8linkuniform sampler2D textures;

9linkuniform float brightValues[100];

10linkuniform float tiles;

11linkuniform float width;

12link

13link

14linkfloat luma(vec3 color) {

15link  return dot(color, vec3(0.299, 0.587, 0.114));

16link}

17link

18linkvoid main() {

19link

20link  vec2 uv = vTexCoord;

21link  vec2 uv2 = vTexCoord;

22link  uv.y = 1.0 - uv.y;

23link  uv2.y = 1.0 - uv2.y;

24link  uv = uv * tiles;

25link

26link  uv = floor(uv);

27link

28link  uv = uv / tiles;

29link

30link

31link  vec4 tex = texture2D(tex0, uv);

32link  float gray = luma(tex.rgb);

33link  vec4 img = texture2D(textures,vec2((uv2.x-uv.x)*tiles/(100.0), (uv2.y-uv.y)*tiles));

34link  float diff = abs(gray - brightValues[0]);

35link  int j = 0;

36link  for (int i = 0; i < 300; i++) {

37link    float newDiff = abs(gray - brightValues[i]);

38link    if (newDiff < diff) {

39link      img = texture2D(textures, vec2((uv2.x-uv.x)*tiles/(100.0)+(float(i)/100.0), (uv2.y-uv.y)*tiles));

40link      diff = newDiff;

41link    }

42link  }

43link

44link  gl_FragColor = img;

45link}

linkComparison with Software implementation

When we measure the framerate of both implementations using the same images as the base dataset we found an improvement 365% in the frameRate, on the software implementation the value of the frameRate was around 4.31 frames per second, while in the hardware implementation we have 20.04 frames per second on average.

Software Image & Video Processingchevron_right

Problem Statement and Background Image and Video filters Image Photographic-Mosaic Ascii art

Kernel (Image processing)chevron_right

Kernel (Image processing) Edge detection Sharpen Blur Unsharp Masking Your own filters

Conclusions and References

Hardware Image & Video Processingchevron_right

Problem Statement and Background Image and Video filters Ascii art Photo Mosaic

Kernel (Image processing)chevron_right

Kernel (Image processing) Edge detection Sharpen Blur Unsharp Masking

Conclusions and References

Renderingchevron_right

Problem Statement and Background Radiosity Algorithm Rasterization using Barycentric coordinates Conclusions and References

Computer Graphics HCI