diff --git a/Brasserie_L_Aficion.png b/Brasserie_L_Aficion.png new file mode 100755 index 0000000..09d1cfc Binary files /dev/null and b/Brasserie_L_Aficion.png differ diff --git a/Brasserie_L_Aficion_0.50_sns.png b/Brasserie_L_Aficion_0.50_sns.png new file mode 100755 index 0000000..df8dcec Binary files /dev/null and b/Brasserie_L_Aficion_0.50_sns.png differ diff --git a/Demo.m b/Demo.m new file mode 100755 index 0000000..3f12539 --- /dev/null +++ b/Demo.m @@ -0,0 +1,58 @@ +% ==== (SNS matlab code)====== +% The Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + +clear all; clc + +im = imread('tajmahal.png'); +im_SNS = imread('tajmahal_0.50_sns.png'); + +% im = imread('Brasserie_L_Aficion.png'); +% im_SNS = imread('Brasserie_L_Aficion_0.50_sns.png'); + +% parameters +Ratio = 0.5; +mesh_size = 20; % using mesh_size x mesh_size quad +[h, w, ~] = size(im); +quad_num_h = floor(h/mesh_size); +quad_num_w = floor(w/mesh_size); + +% the regular mesh on original image +Vertex_set_org = ImgRegualrMeshGrid(im, mesh_size); + +% the importance map generation +importance_map = SNS_importanceMap(im, true); % generate the importance map +importance_quad = SNS_importanceMap_quad(importance_map, Vertex_set_org); +importance_quad = importance_quad/sum(importance_quad(:)); % the importance weight for the quad + +% the naive initialization of the mesh +% retargeting on the width +Vertex_warped_initial = Vertex_set_org; +Vertex_warped_initial(:,:,2) = Vertex_warped_initial(:,:,2)*Ratio; + +% the mesh grid optimization +[Vertex_updated] = ... + SNS_optimization(Vertex_set_org ,Vertex_warped_initial, importance_quad); + +% warp the new image +im_warped = MeshBasedImageWarp(im, [1 Ratio], Vertex_set_org, Vertex_updated); +figure; subplot(1,2,1); imshow(im_warped); title(['My warped'], 'FontSize' , 15); +subplot(1,2,2); imshow(im_SNS); title(['Original SNS warped'], 'FontSize' , 15); + +% show the mesh grid on the original image and retargeted image +% MeshGridImgPlot(im, Vertex_set_org, [0.5 0.0 0.5]); +% title(['Regular mesh grid on original image'], 'FontSize' , 15); +% MeshGridImgPlot(im_warped, Vertex_updated, [0.5 0.0 0.5]); +% title(['Warped image '], 'FontSize' , 15); + + + + diff --git a/ImgRegualrMeshGrid.m b/ImgRegualrMeshGrid.m new file mode 100755 index 0000000..bb6f9bf --- /dev/null +++ b/ImgRegualrMeshGrid.m @@ -0,0 +1,42 @@ +function [ Vertex_set ] = ImgRegualrMeshGrid(im, mesh_size) +% Summary of this function goes here +% Detailed explanation goes here +% The Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + +[h, w, ~] = size(im); +quad_num_h = floor(h/mesh_size); +quad_num_w = floor(w/mesh_size); +Remain_h = h - mesh_size*quad_num_h; +Remain_w = w - mesh_size*quad_num_w; + +Start_point = 0; + +Vertex_set = zeros(quad_num_h+1, quad_num_w+1,2); +for vect_i = 1:quad_num_h+1 + for vect_j = 1:quad_num_w+1 + if(vect_i ~= 1) + Vertex_set(vect_i, vect_j, 1) = ... + min(Start_point+mesh_size*(vect_i-1)+round((vect_i-1)/quad_num_h*Remain_h), h); + else + Vertex_set(vect_i, vect_j, 1) = Start_point; + end + if(vect_j ~= 1) + Vertex_set(vect_i, vect_j, 2) = ... + min(Start_point+mesh_size*(vect_j-1)+round((vect_j-1)/quad_num_w*Remain_w), w); + else + Vertex_set(vect_i, vect_j, 2) = Start_point; + end + end +end + +end + diff --git a/MeshBasedImageWarp.m b/MeshBasedImageWarp.m new file mode 100755 index 0000000..b62d5ed --- /dev/null +++ b/MeshBasedImageWarp.m @@ -0,0 +1,108 @@ +function [im_warped] = MeshBasedImageWarp(im, ratio ,Vertex_set_org, Vertex_set_warped) +% Summary of this function goes here +% Detailed explanation goes here +% The Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + +% ratio is the target ratio [h_ratio, w_ratio] +tic + +[h, w, ~] = size(im); +quad_num_h = size(Vertex_set_org,1) -1; +quad_num_w = size(Vertex_set_org,2) -1; +% GridWarped = NaN(round(h*ratio(1)), round(w*ratio(2)), 2); +GridWarped = zeros(round(h*ratio(1)), round(w*ratio(2)), 2); + +for Quad_h = 1:quad_num_h + for Quad_w = 1:quad_num_w + V1 = [Vertex_set_org(Quad_h, Quad_w, 2) Vertex_set_org(Quad_h, Quad_w, 1)]; + V2 = [Vertex_set_org(Quad_h, Quad_w+1, 2) Vertex_set_org(Quad_h, Quad_w+1, 1)]; + V3 = [Vertex_set_org(Quad_h+1, Quad_w, 2) Vertex_set_org(Quad_h+1, Quad_w, 1)]; + V4 = [Vertex_set_org(Quad_h+1, Quad_w+1, 2) Vertex_set_org(Quad_h+1, Quad_w+1, 1)]; + + V1w = [Vertex_set_warped(Quad_h, Quad_w, 2) Vertex_set_warped(Quad_h, Quad_w, 1)]; + V2w = [Vertex_set_warped(Quad_h, Quad_w+1, 2) Vertex_set_warped(Quad_h, Quad_w+1, 1)]; + V3w = [Vertex_set_warped(Quad_h+1, Quad_w, 2) Vertex_set_warped(Quad_h+1, Quad_w, 1)]; + V4w = [Vertex_set_warped(Quad_h+1, Quad_w+1, 2) Vertex_set_warped(Quad_h+1, Quad_w+1, 1)]; + + Vmatrix = [V1 1; V2 1; V3 1; V4 1]'; + Vwmatrix = [V1w 1; V2w 1; V3w 1; V4w 1]'; + + % compute the projective matrix for the entire quad face + % svd based solution + Amatrix = zeros(8,9); + for i = 1:4 + x = Vmatrix(1,i); y = Vmatrix(2,i); + xw = Vwmatrix(1,i); yw = Vwmatrix(2,i); + Amatrix(2*i-1,:) = [-xw -yw -1 0 0 0 x*xw x*yw x]; + Amatrix(2*i,:) = [0 0 0 -xw -yw -1 y*xw y*yw y]; + end + [~, ~, v] = svd(Amatrix); ColV = v(:,9); + H = [ColV(1:3)'; ColV(4:6)'; ColV(7:9)']/ColV(9); + +% % matrix factorization based solution +% Amatrix = zeros(8,9); +% for i = 1:4 +% x = Vmatrix(1,i); y = Vmatrix(2,i); +% xw = Vwmatrix(1,i); yw = Vwmatrix(2,i); +% Amatrix(2*i-1,:) = [-xw -yw -1 0 0 0 x*xw x*yw x]; +% Amatrix(2*i,:) = [0 0 0 -xw -yw -1 y*xw y*yw y]; +% end +% A_m = Amatrix(:,1:8); B_m = -1*Amatrix(:,9); +% [L, U] = lu(A_m); +% H_foo = U\(L\B_m); +% H = [H_foo(1:3)'; H_foo(4:6)'; H_foo(7:8)' 1]; + + + W_min = min([Vwmatrix(1,:)]); W_max = max([Vwmatrix(1,:)]); + H_min = min([Vwmatrix(2,:)]); H_max = max([Vwmatrix(2,:)]); + + for grid_h = max(floor(H_min),1): ceil(H_max) + for grid_w = max(floor(W_min),1): ceil(W_max) + + if( (grid_w - V1w(1))*(V3w(2) - V1w(2)) - ... + (grid_h - V1w(2))*(V3w(1) - V1w(1)) >= 0 && ... + (grid_w - V2w(1))*(V4w(2) - V2w(2)) - ... + (grid_h - V2w(2))*(V4w(1) - V2w(1)) <= 0 &&... + (grid_w - V1w(1))*(V2w(2) - V1w(2)) - ... + (grid_h - V1w(2))*(V2w(1) - V1w(1)) <= 0 && ... + (grid_w - V3w(1))*(V4w(2) - V3w(2)) - ... + (grid_h - V3w(2))*(V4w(1) - V3w(1)) >= 0); + + grid_point = [grid_w; grid_h; 1]; + grid_backward = H*grid_point; + grid_backward = grid_backward/grid_backward(3); + GridWarped(grid_h, grid_w, 1) = grid_backward(1); + GridWarped(grid_h, grid_w, 2) = grid_backward(2); + end + end + end + + end +end + +% deal with the out-of-border situations +GridWarped( GridWarped<1) = 1; +GridWarped_w = GridWarped(:,:,1); GridWarped_w(GridWarped_w > w) = w; +GridWarped(:,:,1) = GridWarped_w; +GridWarped_h = GridWarped(:,:,2); GridWarped_h(GridWarped_h > h) = h; +GridWarped(:,:,2) = GridWarped_h; + +[X,Y] = meshgrid(1:w,1:h); +im_warped(:,:,1) = interp2(X,Y,double(im(:,:,1)),GridWarped(:,:,1),GridWarped(:,:,2)); +im_warped(:,:,2) = interp2(X,Y,double(im(:,:,2)),GridWarped(:,:,1),GridWarped(:,:,2)); +im_warped(:,:,3) = interp2(X,Y,double(im(:,:,3)),GridWarped(:,:,1),GridWarped(:,:,2)); +im_warped = uint8(im_warped); +time_consumed = toc; +disp(['Warping time : ' num2str(time_consumed) ' s.']); + +end + diff --git a/MeshGridImgPlot.m b/MeshGridImgPlot.m new file mode 100755 index 0000000..0702c42 --- /dev/null +++ b/MeshGridImgPlot.m @@ -0,0 +1,74 @@ +function [ ] = MeshGridImgPlot(im, Vertex_set, COLOR_V) +% Summary of this function goes here +% Detailed explanation goes here +% The Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + +if(nargin < 3) + COLOR_V = [0.0 0.5 0.0]; + % COLOR_V = [0.5 0.0 0.5]; +end + +tic +figure('units' ,'normalized' ,'outerposition' ,[0.01 0.01 0.95 0.95]); +imshow(im); hold on; + +Vertex_set( Vertex_set == 0) = 1; + +[ver_h, ver_w, ~] = size(Vertex_set); +quad_num_h = ver_h - 2; +quad_num_w = ver_w - 2; + +for vect_i = 1:quad_num_h+1 + for vect_j = 1:quad_num_w+1 + pos_ini_x = Vertex_set(vect_i, vect_j, 2); + pos_ini_y = Vertex_set(vect_i, vect_j, 1); + pos_end_x = Vertex_set(vect_i, vect_j+1, 2); + pos_end_y = Vertex_set(vect_i, vect_j+1, 1); + line([pos_ini_x pos_end_x],... + [pos_ini_y pos_end_y],... + 'Color', COLOR_V, 'LineWidth',2); + pos_end_x = Vertex_set(vect_i+1, vect_j, 2); + pos_end_y = Vertex_set(vect_i+1, vect_j, 1); + line([pos_ini_x pos_end_x],... + [pos_ini_y pos_end_y],... + 'Color', COLOR_V, 'LineWidth',2); + + end +end +for vect_i = quad_num_h+2 + for vect_j = 1:quad_num_w+1 + pos_ini_x = Vertex_set(vect_i, vect_j, 2); + pos_ini_y = Vertex_set(vect_i, vect_j, 1); + pos_end_x = Vertex_set(vect_i, vect_j+1, 2); + pos_end_y = Vertex_set(vect_i, vect_j+1, 1); + line([pos_ini_x pos_end_x],... + [pos_ini_y pos_end_y],... + 'Color', COLOR_V, 'LineWidth',2); + end +end +for vect_i = 1:quad_num_h+1 + for vect_j = quad_num_w+2 + pos_ini_x = Vertex_set(vect_i, vect_j, 2); + pos_ini_y = Vertex_set(vect_i, vect_j, 1); + pos_end_x = Vertex_set(vect_i+1, vect_j, 2); + pos_end_y = Vertex_set(vect_i+1, vect_j, 1); + line([pos_ini_x pos_end_x],... + [pos_ini_y pos_end_y],... + 'Color', COLOR_V, 'LineWidth',2); + end +end + +mesh_grid_time = toc; +disp(['Mesh grid plot time consuming: ' num2str(mesh_grid_time) '.']); + +end + diff --git a/Readme.txt b/Readme.txt new file mode 100755 index 0000000..e5ef13b --- /dev/null +++ b/Readme.txt @@ -0,0 +1,18 @@ + +% The Matlab Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + + +The code has been tested on the Windows system. +Available at: http://yabinzhangjohn.github.io/ +To run the code: +1.Install the gbvs: run the gbvs/gbvs_install.m +2.Directly run the Demo.m diff --git a/SNS_Opt_Iter_M.m b/SNS_Opt_Iter_M.m new file mode 100755 index 0000000..11a8174 --- /dev/null +++ b/SNS_Opt_Iter_M.m @@ -0,0 +1,288 @@ +function [ Vertex_updated] = SNS_Opt_Iter_M(Vertex_set_org ,Vertex_warped, importance_quad) +% Summary of this function goes here +% Detailed explanation goes here +% The Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + +[h_V, w_V, ~] = size(Vertex_set_org); + +% compute the sysmetric matrix +% sf update.... +Quad_edge =[0 0 1 0; 0 0 0 1; 0 1 1 1; 1 0 1 1]; +Sf_matrix = zeros(h_V-1, w_V-1); % quad based, each quad one sf; +for Quad_h = 1:h_V-1 + for Quad_w = 1:w_V-1 + for i = 1:4 + v_start = [Vertex_set_org(Quad_h+Quad_edge(1,i), Quad_w+Quad_edge(2,i), 1) ... + Vertex_set_org(Quad_h+Quad_edge(1,i), Quad_w+Quad_edge(2,i), 2)]; + v_end = [Vertex_set_org(Quad_h+Quad_edge(3,i), Quad_w+Quad_edge(4,i), 1) ... + Vertex_set_org(Quad_h+Quad_edge(3,i), Quad_w+Quad_edge(4,i), 2)]; + vw_start = [Vertex_warped(Quad_h+Quad_edge(1,i), Quad_w+Quad_edge(2,i), 1) ... + Vertex_warped(Quad_h+Quad_edge(1,i), Quad_w+Quad_edge(2,i), 2)]; + vw_end = [Vertex_warped(Quad_h+Quad_edge(3,i), Quad_w+Quad_edge(4,i), 1) ... + Vertex_warped(Quad_h+Quad_edge(3,i), Quad_w+Quad_edge(4,i), 2)]; + upper_term(i) = (vw_start- vw_end)*(v_start- v_end)'; + bottom_term(i) = norm(v_start- v_end)^2; + end + Sf_matrix(Quad_h, Quad_w) = sum(upper_term)/sum(bottom_term); + end +end + +% grid line bending term +L_edge_hor = zeros(h_V, w_V-1); +L_edge_ver = zeros(h_V-1, w_V); +% the horizontal edges +for i = 1:h_V + for j = 1:w_V-1 + v_start = [Vertex_set_org(i, j, 1) Vertex_set_org(i, j, 2)]; + v_end = [Vertex_set_org(i, j+1, 1) Vertex_set_org(i, j+1, 2)]; + vw_start = [Vertex_warped(i, j, 1) Vertex_warped(i, j, 2)]; + vw_end = [Vertex_warped(i, j+1, 1) Vertex_warped(i, j+1, 2)]; + foo_l = norm(vw_start - vw_end)/norm(v_start - v_end); + L_edge_hor(i, j) = foo_l; + end +end +% the vertical edges +for i = 1:h_V-1 + for j = 1:w_V + v_start = [Vertex_set_org(i, j, 1) Vertex_set_org(i, j, 2)]; + v_end = [Vertex_set_org(i+1, j, 1) Vertex_set_org(i+1, j, 2)]; + vw_start = [Vertex_warped(i, j, 1) Vertex_warped(i, j, 2)]; + vw_end = [Vertex_warped(i+1, j, 1) Vertex_warped(i+1, j, 2)]; + foo_l = norm(vw_start - vw_end)/norm(v_start - v_end); + L_edge_ver(i, j) = foo_l; + end +end + + +h_Boundary = Vertex_warped(h_V, w_V, 1); +w_Boundary = Vertex_warped(h_V, w_V, 2); + +BENDING_TERM = 1; +Lambda_term = 0.01; + +Vertex_updated = Vertex_warped; +% construct the system matrix +% layer 1 --- H (Y) % layer 2 --- W (X) +for Layer_Vertex = [2 1] % for H and W layer + A_matrix = zeros(h_V*w_V, h_V*w_V); + B_vector = zeros(h_V*w_V, 1); + Vect_vertex_warped_old = reshape(Vertex_warped(:,:,Layer_Vertex), [h_V*w_V, 1]); + for Q_h = 1:h_V + for Q_w = 1:w_V + + Vector_loc = Q_h + (Q_w-1)*h_V; + + % ################################## + % ##### add the quad deformation part coefficients + % 1 === the top-left quad + if( (Q_h - 1) > 0 && (Q_w - 1) > 0) + wf_quad = importance_quad(Q_h - 1, Q_w - 1); + sf_quad = Sf_matrix(Q_h - 1, Q_w - 1); + + A_matrix(Vector_loc, Vector_loc) = ... + A_matrix(Vector_loc, Vector_loc) + 2*wf_quad; + % T + A_matrix(Vector_loc, Vector_loc-1) = ... + A_matrix(Vector_loc, Vector_loc-1) - wf_quad; + % L + A_matrix(Vector_loc, Vector_loc-h_V) = ... + A_matrix(Vector_loc, Vector_loc-h_V) - wf_quad; + % B_vector + B_vector(Vector_loc) = ... + B_vector(Vector_loc) + wf_quad*sf_quad*(2*Vertex_set_org(Q_h,Q_w,Layer_Vertex) ... + - Vertex_set_org(Q_h-1,Q_w,Layer_Vertex) - Vertex_set_org(Q_h,Q_w-1,Layer_Vertex)); + end + + % 2 === the top-right quad + if( (Q_h - 1) > 0 && (Q_w + 1) <= w_V) + wf_quad = importance_quad(Q_h - 1, Q_w); + sf_quad = Sf_matrix(Q_h - 1, Q_w); + + A_matrix(Vector_loc, Vector_loc) = ... + A_matrix(Vector_loc, Vector_loc) + 2*wf_quad; + % T + A_matrix(Vector_loc, Vector_loc-1) = ... + A_matrix(Vector_loc, Vector_loc-1) - wf_quad; + % R + A_matrix(Vector_loc, Vector_loc+h_V) = ... + A_matrix(Vector_loc, Vector_loc+h_V) - wf_quad; + % B_vector + B_vector(Vector_loc) = ... + B_vector(Vector_loc) + wf_quad*sf_quad*(2*Vertex_set_org(Q_h,Q_w,Layer_Vertex) ... + - Vertex_set_org(Q_h-1,Q_w,Layer_Vertex) - Vertex_set_org(Q_h,Q_w+1,Layer_Vertex)); + + end + + % 3 === the bottom-left quad + if( (Q_h + 1) <= h_V && (Q_w - 1) > 0) + wf_quad = importance_quad(Q_h, Q_w - 1); + sf_quad = Sf_matrix(Q_h, Q_w - 1); + + A_matrix(Vector_loc, Vector_loc) = ... + A_matrix(Vector_loc, Vector_loc) + 2*wf_quad; + % D + A_matrix(Vector_loc, Vector_loc+1) = ... + A_matrix(Vector_loc, Vector_loc+1) - wf_quad; + % L + A_matrix(Vector_loc, Vector_loc-h_V) = ... + A_matrix(Vector_loc, Vector_loc-h_V) - wf_quad; + % B_vector + B_vector(Vector_loc) = ... + B_vector(Vector_loc) + wf_quad*sf_quad*(2*Vertex_set_org(Q_h,Q_w,Layer_Vertex) ... + - Vertex_set_org(Q_h+1,Q_w,Layer_Vertex) - Vertex_set_org(Q_h,Q_w-1,Layer_Vertex)); + end + + % 4 === the bottom-right quad + if( (Q_h + 1) <= h_V && (Q_w + 1) <= w_V) + wf_quad = importance_quad(Q_h, Q_w); + sf_quad = Sf_matrix(Q_h, Q_w); + + A_matrix(Vector_loc, Vector_loc) = ... + A_matrix(Vector_loc, Vector_loc) + 2*wf_quad; + % D + A_matrix(Vector_loc, Vector_loc+1) = ... + A_matrix(Vector_loc, Vector_loc+1) - wf_quad; + % R + A_matrix(Vector_loc, Vector_loc+h_V) = ... + A_matrix(Vector_loc, Vector_loc+h_V) - wf_quad; + % B_vector + B_vector(Vector_loc) = ... + B_vector(Vector_loc) + wf_quad*sf_quad*(2*Vertex_set_org(Q_h,Q_w,Layer_Vertex) ... + - Vertex_set_org(Q_h+1,Q_w,Layer_Vertex) - Vertex_set_org(Q_h,Q_w+1,Layer_Vertex)); + end + + % ################################## + if(BENDING_TERM) + % ##### add the grid line bending part coefficients + % 1 === the left edge part (horizontal) + if( Q_w > 1) + L_edge = L_edge_hor(Q_h, Q_w-1); + Nb_H = 0; Nb_W = -1; + + A_matrix(Vector_loc, Vector_loc) = ... + A_matrix(Vector_loc, Vector_loc) + 1*Lambda_term; + % the left vertex + A_matrix(Vector_loc+Nb_H, Vector_loc+Nb_W*h_V) = ... + A_matrix(Vector_loc+Nb_H, Vector_loc+Nb_W*h_V) - 1*Lambda_term; + % B_vector + B_vector(Vector_loc) = B_vector(Vector_loc) + ... + L_edge*(Vertex_set_org(Q_h,Q_w,Layer_Vertex) - ... + Vertex_set_org(Q_h+Nb_H, Q_w+Nb_W,Layer_Vertex))*Lambda_term; + end + % 2 === the top edge part (vertical) + if( Q_h > 1) + L_edge = L_edge_ver(Q_h-1, Q_w); + Nb_H = -1; Nb_W = 0; + + A_matrix(Vector_loc, Vector_loc) = ... + A_matrix(Vector_loc, Vector_loc) + 1*Lambda_term; + % the top vertex + A_matrix(Vector_loc+Nb_H, Vector_loc+Nb_W*h_V) = ... + A_matrix(Vector_loc+Nb_H, Vector_loc+Nb_W*h_V) - 1*Lambda_term; + % B_vector + B_vector(Vector_loc) = B_vector(Vector_loc) + ... + L_edge*(Vertex_set_org(Q_h,Q_w,Layer_Vertex) - ... + Vertex_set_org(Q_h+Nb_H, Q_w+Nb_W,Layer_Vertex) )*Lambda_term; + end + % 3 === the right edge part (horizontal) + if( Q_w < w_V) + L_edge = L_edge_hor(Q_h, Q_w); + Nb_H = 0; Nb_W = 1; + + A_matrix(Vector_loc, Vector_loc) = ... + A_matrix(Vector_loc, Vector_loc) + 1*Lambda_term; + % the right vertex + A_matrix(Vector_loc+Nb_H, Vector_loc+Nb_W*h_V) = ... + A_matrix(Vector_loc+Nb_H, Vector_loc+Nb_W*h_V) - 1*Lambda_term; + % B_vector + B_vector(Vector_loc) = B_vector(Vector_loc) + ... + L_edge*(Vertex_set_org(Q_h,Q_w,Layer_Vertex) - ... + Vertex_set_org(Q_h+Nb_H, Q_w+Nb_W,Layer_Vertex) )*Lambda_term; + end + % 4 === the bottom edge part (vertical) + if( Q_h < h_V) + L_edge = L_edge_ver(Q_h, Q_w); + Nb_H = 1; Nb_W = 0; + + A_matrix(Vector_loc, Vector_loc) = ... + A_matrix(Vector_loc, Vector_loc) + 1*Lambda_term; + % the top vertex + A_matrix(Vector_loc+Nb_H, Vector_loc+Nb_W*h_V) = ... + A_matrix(Vector_loc+Nb_H, Vector_loc+Nb_W*h_V) - 1*Lambda_term; + % B_vector + B_vector(Vector_loc) = B_vector(Vector_loc) + ... + L_edge*(Vertex_set_org(Q_h,Q_w,Layer_Vertex) - ... + Vertex_set_org(Q_h+Nb_H, Q_w+Nb_W,Layer_Vertex) )*Lambda_term; + end + + end + end + + end + + N = 1; + %These constraints are simply substituted into the linear system during the optimization + START_POINT = 0; + if(Layer_Vertex == 1) + for Q_h = 1 + for Q_w = 1:w_V + Vector_loc = Q_h + (Q_w-1)*h_V; + A_matrix(Vector_loc, :) = 0; + A_matrix(Vector_loc, Vector_loc) = 1*N; + B_vector(Vector_loc) = START_POINT*N; + end + end + for Q_h = h_V + for Q_w = 1:w_V + Vector_loc = Q_h + (Q_w-1)*h_V; + A_matrix(Vector_loc, :) = 0; + A_matrix(Vector_loc, Vector_loc) = 1; + B_vector(Vector_loc) = h_Boundary; + end + end + else + for Q_h = 1:h_V + for Q_w = 1 + Vector_loc = Q_h + (Q_w-1)*h_V; + A_matrix(Vector_loc, :) = 0; + A_matrix(Vector_loc, Vector_loc) = 1*N; + B_vector(Vector_loc) = START_POINT*N; + end + end + for Q_h = 1:h_V + for Q_w = w_V + Vector_loc = Q_h + (Q_w-1)*h_V; + A_matrix(Vector_loc, :) = 0; + A_matrix(Vector_loc, Vector_loc) = 1; + B_vector(Vector_loc) = w_Boundary; + end + end + end + + + + [L, U] = lu(A_matrix); + Vect_vertex_warped_factorization = U\(L\B_vector); + + Vect_vertex_warped_factorization = 0.7*Vect_vertex_warped_factorization + ... + 0.3*Vect_vertex_warped_old; + foo = reshape(Vect_vertex_warped_factorization, [h_V w_V]); + Vertex_updated(:, :, Layer_Vertex) = foo; +end + + + + + + +end + diff --git a/SNS_importanceMap.m b/SNS_importanceMap.m new file mode 100755 index 0000000..944421b --- /dev/null +++ b/SNS_importanceMap.m @@ -0,0 +1,114 @@ +function [ importance_map ] = SNS_importanceMap(im, SHOW_MAP) +% Summary of this function goes here +% Detailed explanation goes here +% The Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + + +addpath(genpath('.\gbvs\')); +out_itti = ittikochmap(im); +mapbig = out_itti.master_map_resized; + + + +L = 0.06 * double(im(:,:,1)) + 0.63 * double(im(:,:,2)) + 0.27 * double(im(:,:,3)); +dx = [3 0 -3; 10 0 -10; 3 0 -3]/16; +dy = [3 10 3; 0 0 0; -3 -10 -3]/16; + +IxL1 = conv2(L, dx, 'same'); +IyL1 = conv2(L, dy, 'same'); +GM = sqrt(IxL1.^2 + IyL1.^2); +GM = GM/max(GM(:)); + +% [GM, ~] = imgradient(L,'Sobel'); +% GM = GM/max(GM(:)); + + +importance_map = mapbig.*GM; +% importance_map = GM; +% importance_map = mapbig; + +if(SHOW_MAP) + + colormap_sp = ... + [254 11 1 + 255 26 0 + 254 36 0 + 255 47 0 + 254 60 0 + 254 71 0 + 255 83 1 + 255 96 0 + 254 106 0 + 255 118 1 + 254 131 0 + 255 142 0 + 255 153 2 + 254 166 0 + 254 177 0 + 255 188 0 + 254 201 0 + 255 212 0 + 255 223 1 + 254 237 0 + 254 247 0 + 249 255 0 + 236 255 0 + 225 255 0 + 214 255 0 + 201 255 0 + 190 255 0 + 180 255 0 + 167 255 0 + 155 255 1 + 145 255 0 + 131 255 0 + 120 255 0 + 110 254 0 + 96 255 0 + 85 255 0 + 75 255 0 + 61 255 0 + 50 255 0 + 40 254 0 + 26 255 0 + 16 255 0 + 4 255 0 + 0 247 7 + 0 236 16 + 0 224 27 + 1 210 45 + 0 199 55 + 0 188 64 + 0 173 79 + 0 162 91 + 0 151 100 + 1 139 115 + 0 128 125 + 0 116 137 + 1 102 154 + 0 91 163 + 0 81 173 + 0 66 189 + 1 55 199 + 0 43 211 + 0 29 225 + 0 19 236 + 0 6 249]; + + colormap_sp = double(colormap_sp)/255; + colormap_sp = flipud(colormap_sp); + figure('units' ,'normalized' ,'outerposition' ,[0.01 0.01 0.95 0.95]); + imagesc(importance_map); axis equal; axis off; colormap(colormap_sp); % colorbar +end + +end + diff --git a/SNS_importanceMap_quad.m b/SNS_importanceMap_quad.m new file mode 100755 index 0000000..c124e91 --- /dev/null +++ b/SNS_importanceMap_quad.m @@ -0,0 +1,32 @@ +function [ importance_quad] = SNS_importanceMap_quad(importance_map, Vertex_set_org) +% Summary of this function goes here +% Detailed explanation goes here +% The Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + + quad_num_h = size(Vertex_set_org,1) -1; + quad_num_w = size(Vertex_set_org,2) -1; + importance_quad = zeros(quad_num_h, quad_num_w); + + % layer 1 --- H (Y) % layer 2 --- W (X) + for Quad_h = 1:quad_num_h + for Quad_w = 1:quad_num_w + V1 = [Vertex_set_org(Quad_h, Quad_w, 2) Vertex_set_org(Quad_h, Quad_w, 1)]; + V2 = [Vertex_set_org(Quad_h+1, Quad_w+1, 2) Vertex_set_org(Quad_h+1, Quad_w+1, 1)]; + foo_map = importance_map(max(V1(2),1):V2(2), max(V1(1),1):V2(1)); + + importance_quad(Quad_h, Quad_w) = sum(sum(foo_map)); + end + end + + +end + diff --git a/SNS_optimization.m b/SNS_optimization.m new file mode 100755 index 0000000..77145d1 --- /dev/null +++ b/SNS_optimization.m @@ -0,0 +1,32 @@ +function [ Vertex_updated ] = SNS_optimization(Vertex_set_org, Vertex_warped_initial, importance_quad) +% Summary of this function goes here +% Detailed explanation goes here% The Code (Version 1) is created by ZHANG Yabin, +% Nanyang Technological University, 2015-12-30 +% which is based on the method described in the following paper +% [1] Wang, Yu-Shuen, et al. "Optimized scale-and-stretch for image resizing." +% ACM Transactions on Graphics (TOG) 27.5 (2008): 118. +% The binary code is provided on the project page: +% http://graphics.csie.ncku.edu.tw/Image_Resizing/ +% The Matlab codes are for non-comercial use only. +% Note that the importance maps are slightly different from the original +% ones, and the retargeted images are influenced. + +[Vertex_updated] = ... + SNS_Opt_Iter_M(Vertex_set_org ,Vertex_warped_initial, importance_quad); + +Vertex_updated_old = Vertex_updated; + +Vertex_max_move = inf; +Iter_NUM = 1; +while(Vertex_max_move > 0.5) + Iter_NUM = Iter_NUM + 1; + disp(['########## Iteration no. ' num2str(Iter_NUM)]); + [Vertex_updated] = ... + SNS_Opt_Iter_M(Vertex_set_org ,Vertex_updated, importance_quad); + Vertex_max_move = max(max(max(abs(Vertex_updated_old - Vertex_updated)))); + Vertex_updated_old = Vertex_updated; + disp(['--- Max movement = ' num2str(Vertex_max_move, '%.3f')]); +end + +end + diff --git a/gbvs/algsrc/connectMatrix.m b/gbvs/algsrc/connectMatrix.m new file mode 100755 index 0000000..9e61d5a --- /dev/null +++ b/gbvs/algsrc/connectMatrix.m @@ -0,0 +1,75 @@ +function cx = connectMatrix( dims , lx , inter_type , intra_type , cyclic_type ) + +% inter_type : +% 1 => only neighbor +% 2 => everywhere inter-scale on consecutive scales +% intra_type : +% 1 => only neighbor +% 2 => connect to everynode on same scale +% cyclic_type +% 1 => cyclic boundary rules +% 2 => non-cyclic boundaries + +%% some useful numbers to access nodes at each resolution level +d = prod(dims,2); +N = sum(d); +cx = zeros(N,N); +Nmaps = size(dims,1); +offsets = zeros(Nmaps,1); +for i=2:Nmaps + offsets(i) = d(i-1) + offsets(i-1); +end + +%% connect nodes on a single resolution/level +for i=1:Nmaps + mapsize = d(i); + if ( intra_type == 1 ) %% only neighbors and self + dmatrix = simpledistance( dims(i,:) , cyclic_type ); + dmatrix = (dmatrix <= 1); + cx( (offsets(i)+1):(offsets(i)+mapsize), (offsets(i)+1):(offsets(i)+mapsize) ) = dmatrix; + else %% connect everyone on + cx( (offsets(i)+1):(offsets(i)+mapsize), (offsets(i)+1):(offsets(i)+mapsize) ) = 1; + end +end + +%%% +%% for inter-scale nodes , connect according to some rule +%% inter_type is 1 ==> connect only nodes corresponding to same location +%% inter_type is 2 ==> connect nodes corresponding to different locations +%%% + +map_pairs = mycombnk( [ 1 : Nmaps ] , 2); +for i=1:size(map_pairs,1) + + mapi1 = map_pairs(i,1); + mapi2 = map_pairs(i,2); + + %% nodes in map at level mapi1 + nodes_a = [ offsets(mapi1) + 1 : offsets(mapi1) + d(mapi1) ]; + + %% nodes in map at level mapi2 + nodes_b = [ offsets(mapi2) + 1 : offsets(mapi2) + d(mapi2) ]; + + %% for each pair , possibly connect + for ii=1:d(mapi1) + for jj=1:d(mapi2) + %% using location matrix, determine locations of the + %% two nodes + la = lx( nodes_a(ii) , 3:(2+lx(nodes_a(ii),2)) ); + lb = lx( nodes_b(jj) , 3:(2+lx(nodes_b(jj),2)) ); + + if ( inter_type == 1 ) %% only connect inter-scale nodes + %% which correspond to same location + if ( length( intersect( la , lb ) ) > 0 ) + cx( nodes_a(ii) , nodes_b(jj) ) = 1; + cx( nodes_b(jj) , nodes_a(ii) ) = 1; + end + elseif ( inter_type == 2) %% connect all inter-scale nodes + cx( nodes_a(ii) , nodes_b(jj) ) = 1; + cx( nodes_b(jj) , nodes_a(ii) ) = 1; + end + end % end jj=1:nb + end % end ii=1:na +end % end i=1:size(map_pairs,1) + +cx(cx==0) = inf; diff --git a/gbvs/algsrc/distanceMatrix.m b/gbvs/algsrc/distanceMatrix.m new file mode 100755 index 0000000..cf6d007 --- /dev/null +++ b/gbvs/algsrc/distanceMatrix.m @@ -0,0 +1,74 @@ +function dx = distanceMatrix( dims , lx , cyclic_type ) + +%% some useful numbers to access nodes at each resolution level +d = prod(dims,2); +N = sum(d); +dx = zeros(N,N); +Nmaps = size(dims,1); +offsets = zeros(Nmaps,1); +for i=2:Nmaps + offsets(i) = d(i-1) + offsets(i-1); +end + +sd = {}; +for i=1:Nmaps + sd{i} = simpledistance( dims(i,:) , cyclic_type ); +end + +%% form clique on all nodes in same scale +for i=1:Nmaps + mapsize = d(i); + dmatrix = sd{i}; + for ii=1:mapsize + for jj=1:mapsize + dx( offsets(i)+ii, offsets(i)+jj ) = dmatrix(ii,jj) * d(1)/mapsize; + end + end +end + +map_pairs = mycombnk( [ 1 : Nmaps ] , 2); +dmatrix = sd{1}; +for i=1:size(map_pairs,1) + + mapi1 = map_pairs(i,1); + mapi2 = map_pairs(i,2); + + %% nodes in map at level mapi1 + nodes_a = [ offsets(mapi1) + 1 : offsets(mapi1) + d(mapi1) ]; + + %% nodes in map at level mapi2 + nodes_b = [ offsets(mapi2) + 1 : offsets(mapi2) + d(mapi2) ]; + + %% for each pair , possibly connect + for ii=1:d(mapi1) + for jj=1:d(mapi2) + + %% using location matrix, determine locations of the + %% two nodes + la = lx( nodes_a(ii) , 3:(2+lx(nodes_a(ii),2)) ); + lb = lx( nodes_b(jj) , 3:(2+lx(nodes_b(jj),2)) ); + + nla = length(la); + nlb = length(lb); + + %% convention betweeen 0-index and 1-index + la = la + 1; + lb = lb + 1; + + mean_dist = 0; + max_dist = -inf; + for iii=1:nla + for jjj=1:nlb + dd = dmatrix( la(iii) , lb(jjj) ); + mean_dist = mean_dist + dd; + max_dist = max( dd , max_dist ); + end + end + + mean_dist = mean_dist / (nla*nlb); + dx( nodes_a(ii) , nodes_b(jj) ) = mean_dist; %max_dist; %mean_dist; + dx( nodes_b(jj) , nodes_a(ii) ) = mean_dist; %max_dist; %mean_dist; + + end % end jj=1:nb + end % end ii=1:na +end % end i=1:size(map_pairs,1) diff --git a/gbvs/algsrc/formMapPyramid.m b/gbvs/algsrc/formMapPyramid.m new file mode 100755 index 0000000..01ccf1b --- /dev/null +++ b/gbvs/algsrc/formMapPyramid.m @@ -0,0 +1,51 @@ + +% +% for each delta in deltas , adds map +% delta binary orders smaller to A +% stacks dimensions of maps in A +% + +function [ Apyr , dims ] = formMapPyramid( A , deltas ) + +my_eps = 1e-12; + +num_deltas = length(deltas); +max_delta = max(deltas); +num_pyr = 1 + num_deltas; + +dim = [ size(A) num_pyr ]; +Apyr = zeros( dim ); +dims = zeros(num_pyr,2); + +maps = {}; +maps{1}.map = A; +maps{1}.siz = size(A); +last = A; + +for i=1:max_delta; + last = mySubsample( last ); + maps{i+1}.map = last; + maps{i+1}.siz = size(last); +end + +for i=1:max_delta+1 + map = maps{i}.map; + map = mat2gray(map); + if ( max(map(:)) == 0 ) map = map + my_eps; end + maps{i}.map = map; +end + +Apyr(:,:,1) = maps{1}.map; +dims(1,:) = maps{1}.siz; + +if ( size(deltas,1) > size(deltas,2) ) deltas = deltas'; end +i = 1; +for delta=deltas, + i = i + 1; + d = maps{1+delta}.siz; + m = maps{1+delta}.map; + Apyr(1:d(1),1:d(2),i) = m; + dims(i,:) = d; +end + + diff --git a/gbvs/algsrc/getDims.m b/gbvs/algsrc/getDims.m new file mode 100755 index 0000000..74723de --- /dev/null +++ b/gbvs/algsrc/getDims.m @@ -0,0 +1,5 @@ +function [ dims ] = getDims( orig_size , deltas ) +[tmp,dims] = formMapPyramid( ones(orig_size) , deltas ); + + + diff --git a/gbvs/algsrc/graphsalapply.m b/gbvs/algsrc/graphsalapply.m new file mode 100755 index 0000000..bf749f4 --- /dev/null +++ b/gbvs/algsrc/graphsalapply.m @@ -0,0 +1,61 @@ + + +function [Anorm,iters] = graphsalapply( A , frame , sigma_frac, num_iters , algtype , tol ) + +% +% this function is the heart of GBVS. +% * it takes a feature map, forms a graph over its locations, which is either a lattice of a hierachy ("multiresolution")of lattices, +% connects the nodes with weighted edges, and computes the equilibrium distribution over states. +% * the weight assignment rule from node i to node j depends on the 'algtype' + +% algtype algorith type: +% 1 : MM( i->j ) = w*AL(j) [ mass conc ] +% 2 : MM( i->j ) = w*|AL(i)-AL(j)| [ sal diff ] +% 3 : MM( i->j ) = w*|log(AL(i)/AL(j))| [ sal log ] +% 4 : Anorm = A . ^ w [ simple mass concentration ] + +% tol controls a stopping rule on the computation of the equilibrium distribution (principal eigenvector) +% the lower it is, the longer the algorithm runs. + +if ( algtype == 4 ) + Anorm = A .^ 1.5; + iters = 1; + return; +end + +% form a multiresolution pyramid of feature maps according to multilevels +lx = frame.lx; +[ Apyr , dims ] = formMapPyramid( A , frame.multilevels ); + +% get a weight matrix between nodes based on distance matrix +sig = sigma_frac * mean(size(A)); +Dw = exp( -1 * frame.D / (2 * sig^2) ); + +% assign a linear index to each node +AL = mexArrangeLinear( Apyr , dims ); + +% create the state transition matrix between nodes +P = size(lx,1); +MM = zeros( P , P ); + +iters = 0; + +for i=1:num_iters + + % assign edge weights based on distances between nodes and algtype + mexAssignWeights( AL , Dw , MM , algtype ); + + % make it a markov matrix (so each column sums to 1) + mexColumnNormalize( MM ); + + % find the principal eigenvector + [AL,iteri] = principalEigenvectorRaw( MM , tol ); + iters = iters + iteri; + +end + +% collapse multiresolution representation back onto one scale +Vo = mexSumOverScales( AL , lx , prod(size(A)) ); + +% arrange the nodes back into a rectangular map +Anorm = reshape(Vo,size(A)); diff --git a/gbvs/algsrc/graphsalinit.m b/gbvs/algsrc/graphsalinit.m new file mode 100755 index 0000000..50b06c7 --- /dev/null +++ b/gbvs/algsrc/graphsalinit.m @@ -0,0 +1,34 @@ + +% this function creates the weight matrix for making edge weights +% and saves some other constants (like node-in-lattice index) to a 'frame' +% used when the graphs are made from saliency/feature maps. + +% +% edge types (by default, instantiate fully connected graph. +% use inter/intra-type = 1 to connect only to nearest neighbors) +% +% inter_type : +% 1 => only same-location neighbor +% 2 => everywhere inter-scale on consecutive scales +% intra_type : +% 1 => only neighbor +% 2 => everywhere +% cyclic_type : +% 1 => cyclic boundary rules +% 2 => non-cyclic boundaries +% +% jharel 7 / 27 / 06 + +function [frame] = graphsalinit( map_size , multilevels , inter_type , intra_type , cyclic_type ) + +dims = getDims( map_size , multilevels ); +[N,nam] = namenodes(dims); +lx = makeLocationMap( dims , nam , N ); +cx = connectMatrix( dims , lx , inter_type , intra_type , cyclic_type ); +dx = distanceMatrix( dims , lx , cyclic_type ); +D = cx .* dx; + +frame.D = D; +frame.lx = lx; +frame.dims = dims; +frame.multilevels = multilevels; diff --git a/gbvs/algsrc/indexmatrix.m b/gbvs/algsrc/indexmatrix.m new file mode 100755 index 0000000..0614106 --- /dev/null +++ b/gbvs/algsrc/indexmatrix.m @@ -0,0 +1,8 @@ +function ix = indexmatrix( dims ) + +ix = zeros(dims); +p = prod(dims); + +for i=1:p + ix(i) = i; +end diff --git a/gbvs/algsrc/initGBVS.m b/gbvs/algsrc/initGBVS.m new file mode 100755 index 0000000..5f2cab3 --- /dev/null +++ b/gbvs/algsrc/initGBVS.m @@ -0,0 +1,57 @@ +% +% some constants used across different calls to gbvs() +% + +function [grframe,param] = initGBVS(param, imgsize) + +mymessage(param,'initializing....\n'); + +% logical consistency checking of parameters +if ( min(param.levels) < 2 ) + mymessage(param,'oops. cannot use level 1.. trimming levels used\n'); + param.levels = param.levels(param.levels>1); +end +if ( param.useIttiKochInsteadOfGBVS ) + param.activationType = 2; + param.normalizationType = 3; + param.normalizeTopChannelMaps = 1; +end + +param.maxcomputelevel = max(param.levels); +if (param.activationType==2) + param.maxcomputelevel = max( param.maxcomputelevel , max(param.ittiCenterLevels)+max(param.ittiDeltaLevels) ); +end + +w = imgsize(2); h = imgsize(1); scale = param.salmapmaxsize / max(w,h); +salmapsize = round( [ h w ] * scale ); + +% weight matrix +if ( ~param.useIttiKochInsteadOfGBVS ) + load mypath; + ufile = sprintf('%s__m%s__%s.mat',num2str(salmapsize),num2str(param.multilevels),num2str(param.cyclic_type)); + ufile(ufile==' ') = '_'; + ufile = fullfile( pathroot , 'initcache' , ufile ); + if ( exist(ufile) ) + grframe = load(ufile); + grframe = grframe.grframe; + else + grframe = graphsalinit( salmapsize , param.multilevels , 2, 2, param.cyclic_type ); + save(ufile,'grframe'); + end +else + grframe = []; +end + +% gabor filters +gaborParams.stddev = 2;gaborParams.elongation = 2; +gaborParams.filterSize = -1;gaborParams.filterPeriod = pi; +for i = 1 : length(param.gaborangles) + theta = param.gaborangles(i); + gaborFilters{i}.g0 = makeGaborFilterGBVS(gaborParams, theta, 0); + gaborFilters{i}.g90 = makeGaborFilterGBVS(gaborParams, theta, 90); +end + +param.gaborParams = gaborParams; +param.gaborFilters = gaborFilters; +param.salmapsize = salmapsize; +param.origimgsize = imgsize; diff --git a/gbvs/algsrc/makeLocationMap.m b/gbvs/algsrc/makeLocationMap.m new file mode 100755 index 0000000..2f0e1b6 --- /dev/null +++ b/gbvs/algsrc/makeLocationMap.m @@ -0,0 +1,38 @@ +function lx = makeLocationMap( dims , nam , N ) + +Nmaps = size(dims,1); +lx = zeros(N , 2^13 + 3 ); + +px = {}; +for i = 1 : Nmaps + px{i}.r = partitionindex( dims(1,1) , dims(i,1) ); + px{i}.c = partitionindex( dims(1,2) , dims(i,2) ); +end + +for i = 1 : Nmaps + for j = 1 : dims(i,1) + for k = 1 : dims(i,2) + + nm = nam(j,k,i); + + xcoords = px{i}.r( 1 , find( px{i}.r(2,:) == j ) ); + ycoords = px{i}.c( 1 , find( px{i}.c(2,:) == k ) ); + + lst = []; + Nx = length(xcoords); + Ny = length(ycoords); + Nl = 0; + + for ii=1:Nx + for jj=1:Ny + lst(end+1) = nam( xcoords(ii) , ycoords(jj) , 1 ); + Nl = Nl + 1; + end + end + lx( nm + 1 , 1:Nl+2 ) = [ nm Nl lst ]; + end + end +end + +maxL = max( lx(:,2) ); +lx = lx( : , 1 : maxL + 2 ); diff --git a/gbvs/algsrc/mexArrangeLinear.cc b/gbvs/algsrc/mexArrangeLinear.cc new file mode 100755 index 0000000..e7e5a31 --- /dev/null +++ b/gbvs/algsrc/mexArrangeLinear.cc @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include +#include + +// Avalues = mexArrangeLinear( A , dims ) +// where A is the NxM x K matrix containing multi-resolution info +// dims is K x 2 matrix containing dimensions of each scale +// Avalues is P x 1 matrix containing multi-resolution info in flat array + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + + //Declarations + mxArray *Aar, *dimsar; + double *A, *dims, *Avalues; + + int i,r,c,cur_index; + int N, M, K, P, offset_,M_orig,N_orig,mapsize, coffset; + + // get first argument A + Aar = (mxArray*)prhs[0]; + A = mxGetPr(Aar); + + // get sigma + dimsar = (mxArray*)prhs[1]; + dims = mxGetPr(dimsar); + K = mxGetM(dimsar); // number of rows + + P = 0; + for (i=0;i +#include +#include +#include +#include +#include + +// mexAssignWeights( AL , D , MM , algtype ) +// +// name dim description +// ------------------------------------------- +// AL Px1 values of map linearized +// D PxP w=D(i,j)==D(j,i) is dist multiplier for i & j +// MM PxP output space for markov matrix +// algtype 1x1 algorith type: +// 1 : MM( i->j ) = w*AL(j) [ mass conc ] +// 2 : MM( i->j ) = w*|AL(i)-AL(j)| [ sal diff ] +// 3 : MM( i->j ) = w*|log(AL(i)/AL(j))| [ sal log ] +// 4 : MM( i->j ) = w*1/|AL(i)-AL(j)| [ sal affin ] + +double myabs(double v) { + return (v>=0) ? v : (-1*v); +} + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + + //Declarations + mxArray *ALar, *Dar, *MMar, *algtypear; + double *AL, *D, *MM, *algtype; + double w; + int r, c, P, Pc, Pc_r, algtype_i; + + // get AL + ALar = (mxArray*)prhs[0]; + AL = mxGetPr(ALar); + P = mxGetM(ALar); + + // get D + Dar = (mxArray*)prhs[1]; + D = mxGetPr(Dar); + + // get MM + MMar = (mxArray*)prhs[2]; + MM = mxGetPr(MMar); + + // get algtype + algtypear = (mxArray*)prhs[3]; + algtype = mxGetPr(algtypear); + algtype_i = (int)algtype[0]; + + for (c=0;c +#include +#include +#include +#include +#include + +// Normalizes so that each column sums to one + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + + //Declarations + mxArray *Aar; + double *A; + double s; + int i,j,numR,numC,myoff; + + // get first argument A + Aar = (mxArray*)prhs[0]; + A = mxGetPr(Aar); + numR = mxGetM(Aar); // rows + numC = mxGetN(Aar); // cols + + for (j=0;j +#include +#include +#include +#include +#include + +// Vo = mexSumOverScales( v , lx , N ) +// +// name dim description +// ------------------------------------------------------------------------- +// v P x 1 values of vector linearized +// lx P x (2+K) K = lx(i,2) # of locations corresponding to i +// lx(i,3:3+K) individual locations corresponding to i +// N 1 x 1 # of locations in original size map +// Vo N x 1 components of v summed and collapsed according to lx + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + + // Declarations + mxArray *var, *lxar, *Nar; + double *v, *lx, *Nd, *Vo, vtmp; + int i, j, K , N, P , P2, locum; + + // get v + var = (mxArray*)prhs[0]; + v = mxGetPr(var); + P = mxGetM(var); + + // get lx + lxar = (mxArray*)prhs[1]; + lx = mxGetPr(lxar); + + // get N + Nar = (mxArray*)prhs[2]; + Nd = mxGetPr(Nar); + N = (int)Nd[0]; + + // allocate Vo + plhs[0] = mxCreateDoubleMatrix(N, 1, mxREAL); + Vo = mxGetPr(plhs[0]); + + for (i=0;i +#include +#include +#include +#include +#include + +// outmap = mexVectorToMap( v , dim ) +// +// name dim description +// ------------------------------------------------------------------------- +// v MN x 1 values of map in linear form +// dim 1 x 2 =[M N] dimension of 2D map +// outmap M x N values of map in 2D map form + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + + // Declarations + mxArray *var, *dimar; + double *v, *dim, *outmap; + int i, M, N, MN; + + // get v + var = (mxArray*)prhs[0]; + v = mxGetPr(var); + + // get dim + dimar = (mxArray*)prhs[1]; + dim = mxGetPr(dimar); + M = (int)dim[0]; + N = (int)dim[1]; + MN = M * N; + + // allocate outmap + plhs[0] = mxCreateDoubleMatrix(M, N, mxREAL); + outmap = mxGetPr(plhs[0]); + + for (i=0;i tol ) + oldv = v; + oldoldv = oldv; + v = markovA * v; + df = norm(oldv-v); + iter = iter + 1; + s = sum(v); + if ( s >= 0 && s < inf ) + continue; + else + v = oldoldv; + break; + end +end + +v = v / sum(v); + + diff --git a/gbvs/algsrc/simpledistance.m b/gbvs/algsrc/simpledistance.m new file mode 100755 index 0000000..205d642 --- /dev/null +++ b/gbvs/algsrc/simpledistance.m @@ -0,0 +1,38 @@ +% +% gives you a matrix where +% d( ix(i,j) , ix(ii,jj) ) = distance^2 between (i,j) & (ii,jj) +% +% cyclic_type +% 1 => cyclic boundary rules +% 2 => non-cyclic boundaries + +function d = simpledistance( dim , cyclic_type ) + +d = 0; + +ix = indexmatrix( dim ); +N = prod( dim ); + +d = zeros( N , N ); + +for i=1:dim(1) + for j=1:dim(2) + for ii=1:dim(1) + for jj=1:dim(2) + if ( d( ix(i,j) , ix(ii,jj) ) == 0 ) + di = 0 ; dj = 0; + if ( cyclic_type==1 ) + di = min( abs(i-ii) , abs( abs(i-ii) - dim(1) ) ); + dj = min( abs(j-jj) , abs( abs(j-jj) - dim(2) ) ); + else + di = i-ii; + dj = j-jj; + end + d( ix(i,j) , ix(ii,jj) ) = di^2 + dj^2; + d( ix(ii,jj) , ix(i,j) ) = di^2 + dj^2; + end + end + end + end +end + diff --git a/gbvs/algsrc/sparseness.m b/gbvs/algsrc/sparseness.m new file mode 100755 index 0000000..c5029db --- /dev/null +++ b/gbvs/algsrc/sparseness.m @@ -0,0 +1,2 @@ +function s = sparseness(a) +s = sum(sum(a~=0)) / prod(size(a)); \ No newline at end of file diff --git a/gbvs/compile/cleanmex.m b/gbvs/compile/cleanmex.m new file mode 100755 index 0000000..0b89f1e --- /dev/null +++ b/gbvs/compile/cleanmex.m @@ -0,0 +1 @@ +!rm */*.mex* \ No newline at end of file diff --git a/gbvs/compile/gbvs_compile.m b/gbvs/compile/gbvs_compile.m new file mode 100755 index 0000000..67b8885 --- /dev/null +++ b/gbvs/compile/gbvs_compile.m @@ -0,0 +1,19 @@ + +% cleanmex + +cd util +mex('myContrast.cc'); +cd ../ + +cd algsrc +mex('mexArrangeLinear.cc'); +mex('mexAssignWeights.cc'); +mex('mexColumnNormalize.cc'); +mex('mexSumOverScales.cc'); +mex('mexVectorToMap.cc'); +cd ../ + +cd saltoolbox/ +mex('mySubsample.cc'); +mex('mexLocalMaximaGBVS.cc'); +cd ../ diff --git a/gbvs/compile/gbvs_compile2.m b/gbvs/compile/gbvs_compile2.m new file mode 100755 index 0000000..9efa80d --- /dev/null +++ b/gbvs/compile/gbvs_compile2.m @@ -0,0 +1,19 @@ + +% cleanmex + +cd util +mex -maci64 myContrast.cc ; +cd ../ + +cd algsrc +mex -maci64 mexArrangeLinear.cc ; +mex -maci64 mexAssignWeights.cc ; +mex -maci64 mexColumnNormalize.cc ; +mex -maci64 mexSumOverScales.cc ; +mex -maci64 mexVectorToMap.cc ; +cd ../ + +cd saltoolbox/ +mex -maci64 mySubsample.cc ; +mex -maci64 mexLocalMaximaGBVS.cc ; +cd ../ diff --git a/gbvs/demo/demonstration.m b/gbvs/demo/demonstration.m new file mode 100755 index 0000000..cd2ea61 --- /dev/null +++ b/gbvs/demo/demonstration.m @@ -0,0 +1,73 @@ + +% This file is a demonstration of how to call gbvs() + +% make some parameters +params = makeGBVSParams; + +% could change params like this +params.contrastwidth = .11; + +% example of itti/koch saliency map call +params.useIttiKochInsteadOfGBVS = 1; +outitti = gbvs('samplepics/1.jpg',params); +figure; +subplot(1,2,1); +imshow(imread('samplepics/1.jpg')); +title('image'); +subplot(1,2,2); +imshow(outitti.master_map_resized); +title('Itti, Koch Saliency Map'); +fprintf(1,'Now waiting for user to press enter...\n'); +pause; + +% example of calling gbvs() with default params and then displaying result +outW = 200; +out = {}; +% compute saliency maps for some images +for i = 1 : 5 + + img = imread(sprintf('samplepics/%d.jpg',i)); + + tic; + + % this is how you call gbvs + % leaving out params reset them to all default values (from + % algsrc/makeGBVSParams.m) + out{i} = gbvs( img ); + + toc; + + % show result in a pretty way + + s = outW / size(img,2) ; + sz = size(img); sz = sz(1:2); + sz = round( sz * s ); + + img = imresize( img , sz , 'bicubic' ); + saliency_map = imresize( out{i}.master_map , sz , 'bicubic' ); + if ( max(img(:)) > 2 ) img = double(img) / 255; end + img_thresholded = img .* repmat( saliency_map >= prctile(saliency_map(:),75) , [ 1 1 size(img,3) ] ); + + figure; + subplot(2,2,1); + imshow(img); + title('original image'); + + subplot(2,2,2); + imshow(saliency_map); + title('GBVS map'); + + subplot(2,2,3); + imshow(img_thresholded); + title('most salient (75%ile) parts'); + + subplot(2,2,4); + show_imgnmap(img,out{i}); + title('saliency map overlayed'); + + if ( i < 5 ) + fprintf(1,'Now waiting for user to press enter...\n'); + pause; + end + +end diff --git a/gbvs/demo/flicker_motion_demo.m b/gbvs/demo/flicker_motion_demo.m new file mode 100755 index 0000000..57bbc47 --- /dev/null +++ b/gbvs/demo/flicker_motion_demo.m @@ -0,0 +1,29 @@ + +% some video sequence +i = 1; +for imgi = 85 : 88 + fname{i} = sprintf('samplepics/seq/%03d.jpg',imgi); + i = i + 1; +end +N = length(fname); + +% compute the saliency maps for this sequence + +param = makeGBVSParams; % get default GBVS params +param.channels = 'IF'; % but compute only 'I' instensity and 'F' flicker channels +param.levels = 3; % reduce # of levels for speedup + +motinfo = []; % previous frame information, initialized to empty +for i = 1 : N + [out{i} motinfo] = gbvs( fname{i}, param , motinfo ); +end + +% display results +figure; +for i = 1 : N + subplot(2,N,i); + imshow( imread(fname{i}) ); + title( fname{i} ); + subplot(2,N,N+i); + imshow( out{i}.master_map_resized ); +end diff --git a/gbvs/demo/simplest_demonstration.m b/gbvs/demo/simplest_demonstration.m new file mode 100755 index 0000000..aed372e --- /dev/null +++ b/gbvs/demo/simplest_demonstration.m @@ -0,0 +1,59 @@ + +% example of how to call gbvs with default params + +% img = imread('samplepics/4.jpg'); +% img = All_original_im{1}; +% img = imread('D:\ImageRetargeting\MIT dataset\ArtRoom\ArtRoom.png'); + + +PATH_NAME = {'ArtRoom'; 'BedRoom'; 'Brasserie_L_Aficion'; ... + 'DKNYgirl'; 'Deck'; 'Fatem'; 'Johanneskirche'; 'Lotus'; 'Marblehead_Mass';... + 'Perissa_Santorini'; 'Sanfrancisco'; 'SetAngle'; 'Umdan'; 'Unazukin';... + 'Woman'; 'bicycle2'; 'brick_house'; 'buddha'; 'butterfly'; 'car1'; 'car';... + 'child'; 'face'; 'family'; 'foliage'; 'getty'; 'girls'; 'glasses'; 'greek_wine';... + 'jon'; 'mnm'; 'obama'; 'painting2'; 'surfers'; 'tajmahal'; 'tower'; 'volleyball'}; +PATH_ROOT = 'D:\ImageRetargeting\MIT dataset\'; +for set_num = 1:37 + path = [PATH_ROOT PATH_NAME{set_num} '\']; + file = dir([path,'*.png']); + imgList{set_num} = imread([path file(1).name]); +end + +% for set_num = 1:35 +% prefix = 'D:\ImageRetargeting\LinCW\data\'; +% type = {'mo'; 'scaling'; 'seam'; 'shift'; 'warp'; 'src'}; +% original_path = [prefix type{6} '\']; +% % Extension name of the input file +% ext_type = '*.jpg'; +% img_list = dir([original_path ext_type]); %src +% original_im = imread([original_path img_list(set_num).name]); +% imgList{set_num} = original_im; +% end + +for i = 4 + img = imgList{i}; + out_gbvs = gbvs(img); + out_itti = ittikochmap(img); + + figure; + subplot(2,3,1); + imshow(img); + title('Original Image'); + + subplot(2,3,2); + show_imgnmap(img,out_gbvs); + title('GBVS map overlayed'); + + subplot(2,3,3); + show_imgnmap(img,out_itti); + title('Itti/Koch map overlayed'); + + + subplot(2,3,5); + imshow( out_gbvs.master_map_resized ); + title('GBVS map'); + + subplot(2,3,6); + imshow(out_itti.master_map_resized); + title('Itti/Koch map'); +end diff --git a/gbvs/gbvs.m b/gbvs/gbvs.m new file mode 100755 index 0000000..da712e8 --- /dev/null +++ b/gbvs/gbvs.m @@ -0,0 +1,221 @@ +function [out,motionInfo] = gbvs(img,param,prevMotionInfo) + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% % +% This computes the GBVS map for an image and puts it in master_map. % +% % +% If this image is part of a video sequence, motionInfo needs to be recycled in a % +% loop, and information from the previous frame/image will be used if % +% "flicker" or "motion" channels are employed. % +% You need to initialize prevMotionInfo to [] for the first frame (see demo/flicker_motion_demo.m) % +% % +% input % +% - img can be a filename, or image array (double or uint8, grayscale or rgb) % +% - (optional) param contains parameters for the algorithm (see makeGBVSParams.m) % +% % +% output structure 'out'. fields: % +% - master_map is the GBVS map for img. (.._resized is the same size as img) % +% - feat_maps contains the final individual feature maps, normalized % +% - map_types contains a string description of each map in feat_map (resp. for each index) % +% - intermed_maps contains all the intermediate maps computed along the way (act. & norm.) % +% which are used to compute feat_maps, which is then combined into master_map % +% - rawfeatmaps contains all the feature maps computed at the various scales % +% % +% Jonathan Harel, Last Revised Aug 2008. jonharel@gmail.com % +% % +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +if ( strcmp(class(img),'char') == 1 ) img = imread(img); end +if ( strcmp(class(img),'uint8') == 1 ) img = double(img)/255; end +if ( (size(img,1) < 128) || (size(img,2) < 128) ) + fprintf(2,'GBVS Error: gbvs() meant to be used with images >= 128x128\n'); + out = []; + return; +end + +if ( (nargin == 1) || (~exist('param')) || isempty(param) ) param = makeGBVSParams; end +[grframe,param] = initGBVS(param,size(img)); + + +if ( (nargin < 3) || (~exist('prevMotionInfo')) ) + prevMotionInfo = []; +end + +if ( param.useIttiKochInsteadOfGBVS ) + mymessage(param,'NOTE: Computing STANDARD Itti/Koch instead of Graph-Based Visual Saliency (GBVS)\n\n'); +end + +%%%% +%%%% STEP 1 : compute raw feature maps from image +%%%% + +mymessage(param,'computing feature maps...\n'); +if ( size(img,3) == 3 ) imgcolortype = 1; else, imgcolortype = 2; end +[rawfeatmaps motionInfo] = getFeatureMaps( img , param , prevMotionInfo ); + +%%%% +%%%% STEP 2 : compute activation maps from feature maps +%%%% + +mapnames = fieldnames(rawfeatmaps); +mapweights = zeros(1,length(mapnames)); +map_types = {}; +allmaps = {}; +i = 0; +mymessage(param,'computing activation maps...\n'); +for fmapi=1:length(mapnames) + mapsobj = eval( [ 'rawfeatmaps.' mapnames{fmapi} ';'] ); + numtypes = mapsobj.info.numtypes; + mapweights(fmapi) = mapsobj.info.weight; + map_types{fmapi} = mapsobj.description; + for typei = 1 : numtypes + if ( param.activationType == 1 ) + for lev = param.levels + mymessage(param,'making a graph-based activation (%s) feature map.\n',mapnames{fmapi}); + i = i + 1; + [allmaps{i}.map,tmp] = graphsalapply( mapsobj.maps.val{typei}{lev} , ... + grframe, param.sigma_frac_act , 1 , 2 , param.tol ); + allmaps{i}.maptype = [ fmapi typei lev ]; + end + else + for centerLevel = param.ittiCenterLevels + for deltaLevel = param.ittiDeltaLevels + mymessage(param,'making a itti-style activation (%s) feature map using center-surround subtraction.\n',mapnames{fmapi}); + i = i + 1; + center_ = mapsobj.maps.origval{typei}{centerLevel}; + sz_ = size(center_); + surround_ = imresize( mapsobj.maps.origval{typei}{centerLevel+deltaLevel}, sz_ , 'bicubic' ); + allmaps{i}.map = (center_ - surround_).^2; + allmaps{i}.maptype = [ fmapi centerLevel deltaLevel ]; + end + end + end + end +end + + +%%%% +%%%% STEP 3 : normalize activation maps +%%%% + +mymessage(param,'normalizing activation maps...\n'); +norm_maps = {}; +for i=1:length(allmaps) + mymessage(param,'normalizing a feature map (%d)... ', i); + if ( param.normalizationType == 1 ) + mymessage(param,' using fast raise to power scheme\n ', i); + algtype = 4; + [norm_maps{i}.map,tmp] = graphsalapply( allmaps{i}.map , grframe, param.sigma_frac_norm, param.num_norm_iters, algtype , param.tol ); + elseif ( param.normalizationType == 2 ) + mymessage(param,' using graph-based scheme\n'); + algtype = 1; + [norm_maps{i}.map,tmp] = graphsalapply( allmaps{i}.map , grframe, param.sigma_frac_norm, param.num_norm_iters, algtype , param.tol ); + else + mymessage(param,' using global - mean local maxima scheme.\n'); + norm_maps{i}.map = maxNormalizeStdGBVS( mat2gray(imresize(allmaps{i}.map,param.salmapsize, 'bicubic')) ); + end + norm_maps{i}.maptype = allmaps{i}.maptype; +end + +%%%% +%%%% STEP 4 : average across maps within each feature channel +%%%% + +comb_norm_maps = {}; +cmaps = {}; +for i=1:length(mapnames), cmaps{i}=0; end +Nfmap = cmaps; + +mymessage(param,'summing across maps within each feature channel.\n'); +for j=1:length(norm_maps) + map = norm_maps{j}.map; + fmapi = norm_maps{j}.maptype(1); + Nfmap{fmapi} = Nfmap{fmapi} + 1; + cmaps{fmapi} = cmaps{fmapi} + map; +end +%%% divide each feature channel by number of maps in that channel +for fmapi = 1 : length(mapnames) + if ( param.normalizeTopChannelMaps) + mymessage(param,'Performing additional top-level feature map normalization.\n'); + if ( param.normalizationType == 1 ) + algtype = 4; + [cmaps{fmapi},tmp] = graphsalapply( cmaps{fmapi} , grframe, param.sigma_frac_norm, param.num_norm_iters, algtype , param.tol ); + elseif ( param.normalizationType == 2 ) + algtype = 1; + [cmaps{fmapi},tmp] = graphsalapply( cmaps{fmapi} , grframe, param.sigma_frac_norm, param.num_norm_iters, algtype , param.tol ); + else + cmaps{fmapi} = maxNormalizeStdGBVS( cmaps{fmapi} ); + end + end + comb_norm_maps{fmapi} = cmaps{fmapi}; +end + +%%%% +%%%% STEP 5 : sum across feature channels +%%%% + +mymessage(param,'summing across feature channels into master saliency map.\n'); +master_idx = length(mapnames) + 1; +comb_norm_maps{master_idx} = 0; +for fmapi = 1 : length(mapnames) + mymessage(param,'adding in %s map with weight %0.3g (max = %0.3g)\n', map_types{fmapi}, mapweights(fmapi) , max( cmaps{fmapi}(:) ) ); + comb_norm_maps{master_idx} = comb_norm_maps{master_idx} + cmaps{fmapi} * mapweights(fmapi); +end +master_map = comb_norm_maps{master_idx}; +master_map = attenuateBordersGBVS(master_map,4); +master_map = mat2gray(master_map); + +%%%% +%%%% STEP 6: blur for better results +%%%% +blurfrac = param.blurfrac; +if ( param.useIttiKochInsteadOfGBVS ) + blurfrac = param.ittiblurfrac; +end +if ( blurfrac > 0 ) + mymessage(param,'applying final blur with with = %0.3g\n', blurfrac); + k = mygausskernel( max(size(master_map)) * blurfrac , 2 ); + master_map = myconv2(myconv2( master_map , k ),k'); + master_map = mat2gray(master_map); +end + +if ( param.unCenterBias ) + invCB = load('invCenterBias'); + invCB = invCB.invCenterBias; + centerNewWeight = 0.5; + invCB = centerNewWeight + (1-centerNewWeight) * invCB; + invCB = imresize( invCB , size( master_map ) ); + master_map = master_map .* invCB; + master_map = mat2gray(master_map); +end + +%%%% +%%%% save descriptive, rescaled (0-255) output for user +%%%% + +feat_maps = {}; +for i = 1 : length(mapnames) + feat_maps{i} = mat2gray(comb_norm_maps{i}); +end + +intermed_maps = {}; +for i = 1 : length(allmaps) + allmaps{i}.map = mat2gray( allmaps{i}.map ); + norm_maps{i}.map = mat2gray( norm_maps{i}.map ); +end + +intermed_maps.featureActivationMaps = allmaps; +intermed_maps.normalizedActivationMaps = norm_maps; +master_map_resized = mat2gray(imresize(master_map,[size(img,1) size(img,2)])); + +out = {}; +out.master_map = master_map; +out.master_map_resized = master_map_resized; +out.top_level_feat_maps = feat_maps; +out.map_types = map_types; +out.intermed_maps = intermed_maps; +out.rawfeatmaps = rawfeatmaps; +out.paramsUsed = param; +if ( param.saveInputImage ) + out.inputimg = img; +end diff --git a/gbvs/gbvs_fast.m b/gbvs/gbvs_fast.m new file mode 100755 index 0000000..9074e29 --- /dev/null +++ b/gbvs/gbvs_fast.m @@ -0,0 +1,16 @@ +%% +%% Use this instead of gbvs() if you want slightly less predictive maps +%% computed in a fraction of the time. +%% +%% + +function out = gbvs_fast( img ) + +params = makeGBVSParams; +params.channels = 'DO'; +params.gaborangles = [ 0 90 ]; +params.levels = 3; +params.verbose = 0; +params.tol = 0.003; +params.salmapmaxsize = 24; +out = gbvs(img,params); diff --git a/gbvs/gbvs_install.m b/gbvs/gbvs_install.m new file mode 100755 index 0000000..f3a15ee --- /dev/null +++ b/gbvs/gbvs_install.m @@ -0,0 +1,5 @@ + +pathroot = pwd; +save -mat util/mypath.mat pathroot +addpath(genpath( pathroot ), '-begin'); +savepath \ No newline at end of file diff --git a/gbvs/initcache/17__24__m__2.mat b/gbvs/initcache/17__24__m__2.mat new file mode 100755 index 0000000..d2955f0 Binary files /dev/null and b/gbvs/initcache/17__24__m__2.mat differ diff --git a/gbvs/initcache/18__24__m__2.mat b/gbvs/initcache/18__24__m__2.mat new file mode 100755 index 0000000..c1fc515 Binary files /dev/null and b/gbvs/initcache/18__24__m__2.mat differ diff --git a/gbvs/initcache/18__32__m__2.mat b/gbvs/initcache/18__32__m__2.mat new file mode 100755 index 0000000..2612274 Binary files /dev/null and b/gbvs/initcache/18__32__m__2.mat differ diff --git a/gbvs/initcache/19__24__m__2.mat b/gbvs/initcache/19__24__m__2.mat new file mode 100755 index 0000000..6d2b6c8 Binary files /dev/null and b/gbvs/initcache/19__24__m__2.mat differ diff --git a/gbvs/initcache/19__32__m__2.mat b/gbvs/initcache/19__32__m__2.mat new file mode 100755 index 0000000..d5f04ed Binary files /dev/null and b/gbvs/initcache/19__32__m__2.mat differ diff --git a/gbvs/initcache/20__32__m__2.mat b/gbvs/initcache/20__32__m__2.mat new file mode 100755 index 0000000..724821e Binary files /dev/null and b/gbvs/initcache/20__32__m__2.mat differ diff --git a/gbvs/initcache/21__32__m__2.mat b/gbvs/initcache/21__32__m__2.mat new file mode 100755 index 0000000..2a39ffa Binary files /dev/null and b/gbvs/initcache/21__32__m__2.mat differ diff --git a/gbvs/initcache/22__32__m__2.mat b/gbvs/initcache/22__32__m__2.mat new file mode 100755 index 0000000..7d70c08 Binary files /dev/null and b/gbvs/initcache/22__32__m__2.mat differ diff --git a/gbvs/initcache/23__24__m__2.mat b/gbvs/initcache/23__24__m__2.mat new file mode 100755 index 0000000..f71fd75 Binary files /dev/null and b/gbvs/initcache/23__24__m__2.mat differ diff --git a/gbvs/initcache/23__32__m__2.mat b/gbvs/initcache/23__32__m__2.mat new file mode 100755 index 0000000..53fde86 Binary files /dev/null and b/gbvs/initcache/23__32__m__2.mat differ diff --git a/gbvs/initcache/24__18__m__2.mat b/gbvs/initcache/24__18__m__2.mat new file mode 100755 index 0000000..799a112 Binary files /dev/null and b/gbvs/initcache/24__18__m__2.mat differ diff --git a/gbvs/initcache/24__23__m__2.mat b/gbvs/initcache/24__23__m__2.mat new file mode 100755 index 0000000..79d4930 Binary files /dev/null and b/gbvs/initcache/24__23__m__2.mat differ diff --git a/gbvs/initcache/24__24__m__2.mat b/gbvs/initcache/24__24__m__2.mat new file mode 100755 index 0000000..88fed3f Binary files /dev/null and b/gbvs/initcache/24__24__m__2.mat differ diff --git a/gbvs/initcache/24__32__m__2.mat b/gbvs/initcache/24__32__m__2.mat new file mode 100755 index 0000000..24f7bea Binary files /dev/null and b/gbvs/initcache/24__32__m__2.mat differ diff --git a/gbvs/initcache/25__32__m__2.mat b/gbvs/initcache/25__32__m__2.mat new file mode 100755 index 0000000..9e7ec5b Binary files /dev/null and b/gbvs/initcache/25__32__m__2.mat differ diff --git a/gbvs/initcache/26__32__m__2.mat b/gbvs/initcache/26__32__m__2.mat new file mode 100755 index 0000000..66abe85 Binary files /dev/null and b/gbvs/initcache/26__32__m__2.mat differ diff --git a/gbvs/initcache/27__32__m__2.mat b/gbvs/initcache/27__32__m__2.mat new file mode 100755 index 0000000..15c979b Binary files /dev/null and b/gbvs/initcache/27__32__m__2.mat differ diff --git a/gbvs/initcache/27__40__m__2.mat b/gbvs/initcache/27__40__m__2.mat new file mode 100755 index 0000000..546176e Binary files /dev/null and b/gbvs/initcache/27__40__m__2.mat differ diff --git a/gbvs/initcache/28__32__m__2.mat b/gbvs/initcache/28__32__m__2.mat new file mode 100755 index 0000000..af4dae5 Binary files /dev/null and b/gbvs/initcache/28__32__m__2.mat differ diff --git a/gbvs/initcache/29__30__m__2.mat b/gbvs/initcache/29__30__m__2.mat new file mode 100755 index 0000000..c0c6720 Binary files /dev/null and b/gbvs/initcache/29__30__m__2.mat differ diff --git a/gbvs/initcache/30__28__m__2.mat b/gbvs/initcache/30__28__m__2.mat new file mode 100755 index 0000000..1e35654 Binary files /dev/null and b/gbvs/initcache/30__28__m__2.mat differ diff --git a/gbvs/initcache/30__29__m__2.mat b/gbvs/initcache/30__29__m__2.mat new file mode 100755 index 0000000..32ad945 Binary files /dev/null and b/gbvs/initcache/30__29__m__2.mat differ diff --git a/gbvs/initcache/30__40__m__2.mat b/gbvs/initcache/30__40__m__2.mat new file mode 100755 index 0000000..2e44b46 Binary files /dev/null and b/gbvs/initcache/30__40__m__2.mat differ diff --git a/gbvs/initcache/32__16__m__2.mat b/gbvs/initcache/32__16__m__2.mat new file mode 100755 index 0000000..3e6e660 Binary files /dev/null and b/gbvs/initcache/32__16__m__2.mat differ diff --git a/gbvs/initcache/32__21__m__2.mat b/gbvs/initcache/32__21__m__2.mat new file mode 100755 index 0000000..4dc9999 Binary files /dev/null and b/gbvs/initcache/32__21__m__2.mat differ diff --git a/gbvs/initcache/32__24__m__2.mat b/gbvs/initcache/32__24__m__2.mat new file mode 100755 index 0000000..d94b005 Binary files /dev/null and b/gbvs/initcache/32__24__m__2.mat differ diff --git a/gbvs/initcache/32__25__m__2.mat b/gbvs/initcache/32__25__m__2.mat new file mode 100755 index 0000000..f4f6e81 Binary files /dev/null and b/gbvs/initcache/32__25__m__2.mat differ diff --git a/gbvs/initcache/32__28__m__2.mat b/gbvs/initcache/32__28__m__2.mat new file mode 100755 index 0000000..30ada8d Binary files /dev/null and b/gbvs/initcache/32__28__m__2.mat differ diff --git a/gbvs/initcache/32__31__m__2.mat b/gbvs/initcache/32__31__m__2.mat new file mode 100755 index 0000000..1ccdbb1 Binary files /dev/null and b/gbvs/initcache/32__31__m__2.mat differ diff --git a/gbvs/initcache/32__32__m__2.mat b/gbvs/initcache/32__32__m__2.mat new file mode 100755 index 0000000..533d3e7 Binary files /dev/null and b/gbvs/initcache/32__32__m__2.mat differ diff --git a/gbvs/initcache/35__40__m__2.mat b/gbvs/initcache/35__40__m__2.mat new file mode 100755 index 0000000..fdb80a6 Binary files /dev/null and b/gbvs/initcache/35__40__m__2.mat differ diff --git a/gbvs/initcache/40__30__m__2.mat b/gbvs/initcache/40__30__m__2.mat new file mode 100755 index 0000000..b3fbdef Binary files /dev/null and b/gbvs/initcache/40__30__m__2.mat differ diff --git a/gbvs/initcache/40__38__m__2.mat b/gbvs/initcache/40__38__m__2.mat new file mode 100755 index 0000000..fd814c5 Binary files /dev/null and b/gbvs/initcache/40__38__m__2.mat differ diff --git a/gbvs/initcache/40__40__m__2.mat b/gbvs/initcache/40__40__m__2.mat new file mode 100755 index 0000000..49a21e6 Binary files /dev/null and b/gbvs/initcache/40__40__m__2.mat differ diff --git a/gbvs/ittikochmap.m b/gbvs/ittikochmap.m new file mode 100755 index 0000000..d3fb17e --- /dev/null +++ b/gbvs/ittikochmap.m @@ -0,0 +1,21 @@ +function out = ittikochmap( img ) + +params = makeGBVSParams; +params.useIttiKochInsteadOfGBVS = 1; +params.channels = 'CIO'; +params.verbose = 1; +params.unCenterBias = 0; + +% +% uncomment the line below (ittiDeltaLevels = [2 3]) for more faithful implementation +% (however, known to give crappy results for small images i.e. < 640 in height or width ) +% +% params.ittiDeltaLevels = [ 2 3 ]; +% + +if ( strcmp(class(img),'char') == 1 ) img = imread(img); end +if ( strcmp(class(img),'uint8') == 1 ) img = double(img)/255; end + +params.salmapmaxsize = round( max(size(img))/8 ); + +out = gbvs(img,params); diff --git a/gbvs/makeGBVSParams.m b/gbvs/makeGBVSParams.m new file mode 100755 index 0000000..d67ebd0 --- /dev/null +++ b/gbvs/makeGBVSParams.m @@ -0,0 +1,141 @@ +function p = makeGBVSParams() + +p = {}; + +%%%%%%%%%%%%% general %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +p.salmapmaxsize = 32; % size of output saliency maps (maximum dimension) + % don't set this too high (e.g., >60) + % if you want a saliency map at the + % original image size, just used rescaled + % saliency map + % (out.master_map_resized in gbvs()) + +p.verbose = 0; % turn status messages on (1) / off (0) +p.verboseout = 'screen'; % = 'screen' to echo messages to screen + % = 'myfile.txt' to echo messages to file + +p.saveInputImage = 0; % save input image in output struct + % (can be convenient, but is wasteful + % to store uncompressed image data + % around) + +p.blurfrac = 0.02; % final blur to apply to master saliency map + % (in standard deviations of gaussian kernel, + % expressed as fraction of image width) + % Note: use value 0 to turn off this feature. + +%%%%%%%%%%%%% feature channel parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +p.channels = 'DIO'; % feature channels to use encoded as a string + % these are available: + % C is for Color + % I is for Intensity + % O is for Orientation + % R is for contRast + % F is for Flicker + % M is for Motion + % D is for DKL Color (Derrington Krauskopf Lennie) == + % much better than C channel + % e.g., 'IR' would be only intensity and + % contrast, or + % 'CIO' would be only color,int.,ori. (standard) + % 'CIOR' uses col,int,ori, and contrast + +p.colorWeight = 1; % weights of feature channels (do not need to sum to 1). +p.intensityWeight = 1; +p.orientationWeight = 1; +p.contrastWeight = 1; +p.flickerWeight = 1; +p.motionWeight = 1; +p.dklcolorWeight = 1; + +p.gaborangles = [ 0 45 90 135 ]; % angles of gabor filters +p.contrastwidth = .1; % fraction of image width = length of square side over which luminance variance is + % computed for 'contrast' feature map + % LARGER values will give SMOOTHER + % contrast maps + +p.flickerNewFrameWt = 1; % (should be between 0.0 and 1.0) + % The flicker channel is the abs() difference + % between the *previous frame estimate* and + % current frame. + % This parameter is the weight used + % to update the previous frame estimate. + % 1 == set previous frame to current + % frame + % w == set previous frame to w * present + % + (1-w) * previous estimate + +p.motionAngles = [ 0 45 90 135 ]; + % directions of motion for motion channel + % --> 0 , /^ 45 , |^ 90 , ^\ 135 , etc. + % question: should use more directions? + % e.g., 180, 225, 270, 315, ? + +%%%%%%%%%%%%% GBVS parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +p.unCenterBias = 0; % turned off (0) by default. Attempts to undo some emergent + % center bias in GBVS (by pointwise-multiplying final saliency map by + % an inverse-bias map). + +p.levels = [2 3 4]; % resolution of feature maps relative to original image (in 2^-(n-1) fractions) + % (default [ 2 3 4]) .. maximum level 9 is allowed + % these feature map levels will be used + % if graph-based activation is used. + % otherwise, the ittiCenter/Delta levels + % are (see below) + % minimum value allowed = 2 + % maximum value allowed = 9 + +p.multilevels = []; % [1 2] corresponds to 2 additional node lattices , + % ... one at half and one at quarter size + % use [] for single-resolution version of algorithm. + +p.sigma_frac_act = 0.15; % sigma parameter in activation step of GBVS (as a fraction of image width) - default .15 +p.sigma_frac_norm = 0.06; % sigma parameter in normalizaiton step of GBVS (as a fraction of image width) - default .06 +p.num_norm_iters = 1; % number of normalization iterations in GBVS - default 1 + +p.tol = .0001; % tol controls a stopping rule on the computation of the equilibrium distribution (principal eigenvector) + % the higher it is, the faster the algorithm runs, but the more approximate it becomes. + % it is used by algsrc/principalEigenvectorRaw.m - default .0001 + + +p.cyclic_type = 2; % this should *not* be changed (non-cyclic boundary rules) + +%%%%%%%%%% Parameters to use Itti/Koch and/or Simpler Saliency Algorith %%%% + + +p.useIttiKochInsteadOfGBVS = 0; % use value '0' for Graph-Based Visual Saliency + % use value '1' for Itti Koch algorithm: + % "A Model of Saliency-Based Visual + % Attention for Rapid Scene Analysis", + % PAMI 1998 + +p.activationType = 1; % 1 = graph-based activation (default) + % 2 = center-surround activation (given + % by ittiCenter/DeltaLevels below) + % ( type 2 used if useIttiKoch..= 1 ) + +p.normalizationType = 1; % 1 = simplest & fastest. raises map values to a power before adding them together (default) + % 2 = graph-based normalization scheme (no longer recommended) + % 3 = normalization by (M-m)^2, where M = + % global maximum. m = avg. of local + % maxima + % ( type 3 used if useIttiKoch..=1 ) + +p.normalizeTopChannelMaps = 0; % this specifies whether to normalize the + % top-level feature map of each + % channel... (in addition to normalizing + % maps across scales within a channel) + % 0 = don't do it (default) + % 1 = do it. (used by ittiKoch scheme) + +p.ittiCenterLevels = [ 2 3 ]; % the 'c' scales for 'center' maps + +p.ittiDeltaLevels = [ 2 ]; % the 'delta' in s=c+delta levels for 'surround' scales + % NOTE: for more faithful implementation , use deltaLevels = [ 2 3 ], not [ 2 ] + % however, 3 can be problematic for images < 640 in width or height + +p.ittiblurfrac = 0.03; % apply final blur to master saliency map + % (not in original Itti/Koch algo. but improves eye-movement predctions) \ No newline at end of file diff --git a/gbvs/readme.txt b/gbvs/readme.txt new file mode 100755 index 0000000..33d9d67 --- /dev/null +++ b/gbvs/readme.txt @@ -0,0 +1,210 @@ + +Graph-Based Visual Saliency (MATLAB source code) +http://www.klab.caltech.edu/~harel/share/gbvs.php + +Jonathan Harel +jonharel@gmail.com +California Institute of Technology + +======================================================================================== + +This is an installation and general help file for the saliency map MATLAB code here. + +======================================================================================== + +What you can do with this code: + +(1) Compute a "Graph-Based Visual Saliency" map for an image or image sequence (video) + (as described in J. Harel, C. Koch, and P. Perona. "Graph-Based Visual Saliency", + NIPS 2006 + http://www.klab.caltech.edu/~harel/pubs/gbvs_nips.pdf) + +(2) Compute the standard Itti, Koch, Niebur (PAMI 1998) saliency map. + +(3) Compute modified versions of the above by altering the input parameters. + +======================================================================================== + +Step-by-step start-up procedure: + +(1) Add gbvs to your path: + + Change into the directory containing this file, and enter at the matlab prompt: + + >> gbvs_install + + If you are on a shared machine, you may get an error message such as: + + Warning: Unable to save path to file '/opt/matlab/toolbox/local/pathdef.m' + In savepath at 162 + In gbvs_install at 5 + + In that case, comment out the savepath (i.e., 'savepath' => '% savepath') + command in gbvs_install.m, and add this line to your startup.m file: + + run ???/gbvs_install + + where "???" is replaced by the main gbvs/ directory, which contains the + gbvs_install function + +(2) Now you are ready to compute GBVS maps: + + Demonstrations: + + >> simplest_demonstration + + see demo/demonstration.m for more complicated demo or run: + [Note: if you get an error, see point (3) below] + + >> demonstration + + Basic Usage Example: + + >> out = gbvs( 'samplepics/1.jpg' ); + + You can also compute an Itti/Koch map as follows: + + >> out = ittikochmap( 'samplepics/1.jpg' ); + + Or, to call GBVS simplified to some extent (e.g. no Orientation channel) so that it runs faster, use + + >> out = gbvs_fast( 'samplepics/1.jpg'); + + Now, out.master_map contains your saliency map, and out.master_map_resized is + this saliency map interpolated (bicubic) to the resolution of the original + image. + + For video (not static images): + You need to pass into gbvs() previous frame information, which is returned + on output at every call to gbvs(). + + See demo/flicker_motion_demo.m + + Here is the heart of it: + + motinfo = []; % previous frame information, initialized to empty + for i = 1 : N + [out{i} motinfo] = gbvs( fname{i}, param , motinfo ); + end + +(3) If you are not on 32 or 64 bit Windows, or on Intel-based Mac, or 32 or 64 bit Linux, + and calling simplest_demonstration results in an error, you may have to compile + a few .cc source code files into binary "mex" format. + + You can do that as follows. From the gbvs/ directory, in matlab, run: + + >> gbvs_compile + + If this works properly, there should be no output at all, and you're done! + Then go back to step (2), i.e. try running the demonstration. + + Error note: + If this is your first time compiling mex files, you may have to run: + + >> mex -setup + + and follow the instructions (typically, enter a number, to select a co- + mpiler. then you can run "gbvs_compile"; if it doesn't work, run + "mex -setup" again to select a different compiler, run "gbvs_compile" + again, etc.) + +======================================================================================== + +Helpful Notes: + +(1) inputs of gbvs(): + + * the first argument to gbvs() can be an image name or image array + * there is an optional, second, parameters argument + +(2) outputs of gbvs(): + + * all put into a single structure with various descriptive fields. + * the GBVS map: master_map + (interpolated to the resolution of the input image: master_map_resized) + * master saliency map for each channel: feat_maps (and their names, + map_types) + * all intermediate maps to create the previous two (intermed_maps). see + gbvs.m for details + +(3) the parameter argument: + + * initialized by makeGBVSParams.m -- read that for details. + + Some very sparse notes on fields of the parameter argument: + + sigma_frac_act controls the spatial spread of the function modulating + weights between different image locations (in image widths). + greater value means greater connectivity between distant + locations. + + tol tolerance parameter. governs how accurately the princi- + pal eigenvector calculation is performed. change it to + higher values to make things run faster. + + levels the resolution of the feature maps used to compute the + final master map, relative to the original image size + +(4) Notes on feature maps: + + * are produced by util/getFeatureMaps.m + + * by default, color, intensity, orientation maps are computed. + + which channels are used is controlled by the parameters argument. in part- + icular, you can choose which of these is included by editing the + params.channels string (see makeGBVSParams.m). you can set + their relative weighting also in the parameters. + + If you want to introduce a new feature channel, put a new function into + util/featureChannels/ . Make sure to edit the channels string appropria- + tely. Follow pattern of other channels for proper implementation. + +(5) If you want to compare saliency maps to fixations (e.g., inferred from + scanpaths recorded by an eye-tracker), use: + + >> score = rocScoreSaliencyVsFixations(salmap,X,Y,origimgsize) + + This outputs ROC Area-Under-Curve Score between a saliency map and fixat- + ions. + + salmap : a saliency map + X : vector of X locations of fixations in original image + Y : vector of Y locations of fixations in original image + origimgsize : size of original image (should have same aspect ratio as + saliency map) + +======================================================================================== + +Credits: + +(1) saltoolbox/ directory -- adapted from: Dirk Walther, http://www.saliencytoolbox.net + +(2) Thanks to Alexander G. Huth for help with making heatmap_overlay.m readable. + +======================================================================================== + +Revision History + +first authored 8/31/2006 +Revised 4/25/2008 +Revised 6/5/2008 +Revised 6/26/2008 + added Itti/Koch algorithm +Revised 8/25/2008 + added Flicker/Motion channels +Revised 11/3/2008 + added myconv2 +Revised 2/19/2010 + added initcache to reduce initialization times +Revised 3/18/2010 + added attenuateBordersGBVS to O_orientation call +Revised 1/17/2011 + added attenuateBordersGBVS to master_map. + changed boundary condition in padImage + changed ittiDeltaLevels for ittiKoch to just [2] by default + removed Intensity channel from gbvs_fast +Revised 10/24/2011 + added unCenterBias to parameters, turned it on by default +Revised 7/24/2012 + show_imgnmap returns output. for win users: initGBVS uses fullfile. diff --git a/gbvs/saltoolbox/attenuateBordersGBVS.m b/gbvs/saltoolbox/attenuateBordersGBVS.m new file mode 100755 index 0000000..b5dc633 --- /dev/null +++ b/gbvs/saltoolbox/attenuateBordersGBVS.m @@ -0,0 +1,35 @@ +function result = attenuateBordersGBVS(data,borderSize) +% attentuateBorders - linearly attentuates the border of data. +% +% result = attenuateBorders(data,borderSize) +% linearly attenuates a border region of borderSize +% on all sides of the 2d data array + +% This file is part of the SaliencyToolbox - Copyright (C) 2006 +% by Dirk Walther and the California Institute of Technology. +% The Saliency Toolbox is released under the GNU General Public +% License. See the enclosed COPYRIGHT document for details. +% For more information about this project see: +% http://www.saliencytoolbox.net + +result = data; +dsz = size(data); + +if (borderSize * 2 > dsz(1)) borderSize = floor(dsz(1) / 2); end +if (borderSize * 2 > dsz(2)) borderSize = floor(dsz(2) / 2); end +if (borderSize < 1) return; end + +bs = [1:borderSize]; +coeffs = bs / (borderSize + 1); + +% top and bottom +rec = repmat(coeffs',1,dsz(2)); +result(bs,:) = result(bs,:) .* rec; +range = dsz(1) - bs + 1; +result(range,:) = result(range,:) .* rec; + +% left and right +rec = repmat(coeffs,dsz(1),1); +result(:,bs) = result(:,bs) .* rec; +range = dsz(2) - bs + 1; +result(:,range) = result(:,range) .* rec; diff --git a/gbvs/saltoolbox/makeGaborFilterGBVS.m b/gbvs/saltoolbox/makeGaborFilterGBVS.m new file mode 100755 index 0000000..188f464 --- /dev/null +++ b/gbvs/saltoolbox/makeGaborFilterGBVS.m @@ -0,0 +1,78 @@ +function filter = makeGaborFilterGBVS(gaborParams, angle, phase, varargin) +% makeGaborFilter - returns a 2d Gabor filter. +% +% filter = makeGaborFilter(gaborParams, angle, phase, makeDisc) +% Returns a two-dimensional Gabor filter with the parameter: +% gaborParams - a struct with the following fields: +% filterPeriod - the period of the filter in pixels, +% elongation - the ratio of length versus width, +% filterSize - the size of the filter in pixels, +% stddev - the standard deviation of the Gaussian in pixels. +% angle - the angle of orientation, in degrees, +% phase - the phase of the filter, in degrees, +% makeDisc - if 1, enforce a disc-shaped filter, i.e. set all values +% outside of a circle with diameter gaborParams.filterSize to 0. +% +% filter = makeGaborFilter(gaborParams, angle, phase) +% Returns a two-dimensional Gabor filter, assuming makeDisc = 0. +% +% See also gaborFilterMap, defaultSaliencyParams. + +% This file is part of the Saliency Toolbox - Copyright (C) 2006 +% by Dirk Walther and the California Institute of Technology. +% The Saliency Toolbox is released under the GNU General Public +% License. See the enclosed COPYRIGHT document for details. +% For more information about this project see: +% http://www.saliencytoolbox.net + +if isempty(varargin) + makeDisc = 0; +else + makeDisc = varargin{1}; +end + +% repare parameters +major_stddev = gaborParams.stddev; +minor_stddev = major_stddev * gaborParams.elongation; +max_stddev = max(major_stddev,minor_stddev); + +sz = gaborParams.filterSize; +if (sz == -1) + sz = ceil(max_stddev*sqrt(10)); +else + sz = floor(sz/2); +end + +psi = pi / 180 * phase; +rtDeg = pi / 180 * angle; + +omega = 2 * pi / gaborParams.filterPeriod; +co = cos(rtDeg); +si = -sin(rtDeg); +major_sigq = 2 * major_stddev^2; +minor_sigq = 2 * minor_stddev^2; + +% prepare grids for major and minor components +vec = [-sz:sz]; +vlen = length(vec); +vco = vec*co; +vsi = vec*si; + +major = repmat(vco',1,vlen) + repmat(vsi,vlen,1); +major2 = major.^2; +minor = repmat(vsi',1,vlen) - repmat(vco,vlen,1); +minor2 = minor.^2; + +% create the actual filter +result = cos(omega * major + psi) .* ... +exp(-major2 / major_sigq ... + -minor2 / minor_sigq); + +% enforce disc shape? +if (makeDisc) + result((major2+minor2) > (gaborParams.filterSize/2)^2) = 0; +end + +% normalization +filter = result - mean(result(:)); +filter = filter / sqrt(sum(filter(:).^2)); diff --git a/gbvs/saltoolbox/maxNormalizeStdGBVS.m b/gbvs/saltoolbox/maxNormalizeStdGBVS.m new file mode 100755 index 0000000..fae4f47 --- /dev/null +++ b/gbvs/saltoolbox/maxNormalizeStdGBVS.m @@ -0,0 +1,39 @@ +function result = maxNormalizeStdGBVS(data) +% maxNormalizeStd - normalization based on local maxima. +% result = maxNormalizeStd(data) +% Normalize data by multiplying it with +% (max(data) - avg(localMaxima))^2 as described in; +% L. Itti, C. Koch, E. Niebur, A Model of Saliency-Based +% Visual Attention for Rapid Scene Analysis, IEEE PAMI, +% Vol. 20, No. 11, pp. 1254-1259, Nov 1998. +% +% result = maxNormalizeStd(data,minmax) +% Specify a dynamic for the initial maximum normalization +% of the input data (default: [0 10]). +% +% See also maxNormalize, maxNormalizeFancy, maxNormalizeFancyFast, makeSaliencyMap. + +% This file is part of the Saliency Toolbox - Copyright (C) 2005 +% by Dirk Walther and the California Institute of Technology. +% The Saliency Toolbox is released under the GNU General Public +% License. See the enclosed COPYRIGHT document for details. +% For more information about this project see: +% http://klab.caltech.edu/~walther/SaliencyToolbox + +% +% modified by jonathan harel 2008 for GBVS code +% .. simplified + +M = 10; + +data = mat2gray( data ) * M; +thresh = M / 10; +[lm_avg,lm_num,lm_sum] = mexLocalMaximaGBVS(data,thresh); + +if (lm_num > 1) + result = data * (M - lm_avg)^2; +elseif (lm_num == 1) + result = data * M .^ 2; +else + result = data; +end diff --git a/gbvs/saltoolbox/mexLocalMaximaGBVS.cc b/gbvs/saltoolbox/mexLocalMaximaGBVS.cc new file mode 100755 index 0000000..df909fb --- /dev/null +++ b/gbvs/saltoolbox/mexLocalMaximaGBVS.cc @@ -0,0 +1,68 @@ +#include +#include +#include +#include +#include +#include + +double getVal(double* img, int x, int y, int w, int h); +void getLocalMaxima(double* img, double thresh, int *lm_num, double *lm_sum, int w, int h); + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) +{ + // input + double* img, *thresh; + + // output + double lm_avg, lm_sum; int lm_num; + + double* tmp; + + img = mxGetPr( prhs[0] ); + thresh = mxGetPr( prhs[1] ); + + getLocalMaxima( img, thresh[0] , &lm_num, &lm_sum , mxGetN(prhs[0]) , mxGetM(prhs[0]) ); + + if (lm_sum > 0) lm_avg = (double)lm_sum / (double)lm_num; + else lm_avg = 0.0; + + plhs[0] = mxCreateDoubleMatrix(1, 1, mxREAL); //mxReal is our data-type + plhs[1] = mxCreateDoubleMatrix(1, 1, mxREAL); //mxReal is our data-type + plhs[2] = mxCreateDoubleMatrix(1, 1, mxREAL); //mxReal is our data-type + + tmp = mxGetPr(plhs[0]); tmp[0] = lm_avg; + tmp = mxGetPr(plhs[1]); tmp[0] = lm_num; + tmp = mxGetPr(plhs[2]); tmp[0] = lm_sum; +} + +double getVal(double* img, int x, int y, int w, int h) +{ + double* ptr = img + x * h + y; + return *ptr; +} + +void getLocalMaxima(double* img, double thresh, int *lm_num, double *lm_sum, int w, int h) +{ + int i,j; + double val; + // then get the mean value of the local maxima: + *lm_sum = 0.0; *lm_num = 0; + + for (j = 1; j < h - 1; j ++) + for (i = 1; i < w - 1; i ++) + { + val = getVal(img,i,j,w,h); + if (val >= thresh && + val >= getVal(img,i-1, j,w,h) && + val >= getVal(img,i+1, j,w,h) && + val >= getVal(img,i, j+1,w,h) && + val >= getVal(img,i, j-1,w,h)) // local max + { + *lm_sum += val; + (*lm_num)++; + } + } + return; +} + + diff --git a/gbvs/saltoolbox/mexLocalMaximaGBVS.mexa64 b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexa64 new file mode 100755 index 0000000..833beb0 Binary files /dev/null and b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexa64 differ diff --git a/gbvs/saltoolbox/mexLocalMaximaGBVS.mexglx b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexglx new file mode 100755 index 0000000..d5b8be2 Binary files /dev/null and b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexglx differ diff --git a/gbvs/saltoolbox/mexLocalMaximaGBVS.mexmaci b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexmaci new file mode 100755 index 0000000..61b07cc Binary files /dev/null and b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexmaci differ diff --git a/gbvs/saltoolbox/mexLocalMaximaGBVS.mexmaci64 b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexmaci64 new file mode 100755 index 0000000..2517ba7 Binary files /dev/null and b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexmaci64 differ diff --git a/gbvs/saltoolbox/mexLocalMaximaGBVS.mexw32 b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexw32 new file mode 100755 index 0000000..97aedf1 Binary files /dev/null and b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexw32 differ diff --git a/gbvs/saltoolbox/mexLocalMaximaGBVS.mexw64 b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexw64 new file mode 100755 index 0000000..c9e569a Binary files /dev/null and b/gbvs/saltoolbox/mexLocalMaximaGBVS.mexw64 differ diff --git a/gbvs/saltoolbox/mySubsample.cc b/gbvs/saltoolbox/mySubsample.cc new file mode 100755 index 0000000..fed70a2 --- /dev/null +++ b/gbvs/saltoolbox/mySubsample.cc @@ -0,0 +1,192 @@ +#include +#include +#include +#include +#include +#include + +void lowPass6yDecY(float* sptr, float* rptr, int w, int hs); +void lowPass6xDecX(float* sptr, float* rptr, int ws, int h); +void double2float(double *a, float* b, int N) { + int i; for (i=0;i 10) && (h > 10) ) { + + imgF = (float*)mxMalloc(sizeof(float)*h*w); + decx = (float*)mxMalloc(sizeof(float)*h*wr); + decxy = (float*)mxMalloc(sizeof(float)*hr*wr); + + imgD = mxGetPr(img); + double2float( imgD, imgF, h*w ); + lowPass6xDecX( imgF , decx, w, h ); + lowPass6yDecY( decx , decxy, wr, h ); + + plhs[0] = mxCreateDoubleMatrix(hr, wr, mxREAL); + outArray = mxGetPr(plhs[0]); + float2double( decxy, outArray, hr * wr ); + + mxFree(imgF); + mxFree(decx); + mxFree(decxy); + + } else { + plhs[0] = mxCreateDoubleMatrix(h, w, mxREAL); + outArray = mxGetPr(plhs[0]); + memcpy( outArray , mxGetPr(img) , sizeof(double) * h * w ); + } + +} + +// ###################################################################### +// kernel: 1 5 10 10 5 1 +void lowPass6yDecY(float* sptr, float* rptr, int w, int hs) +{ + int x, y; + int hr = hs / 2; + if (hr == 0) hr = 1; + + /* if (hs <= 1) + result = src; + else + */ + if (hs == 2) + for (x = 0; x < w; ++x) + { + // use kernel [1 1]^T / 2 + *rptr++ = (sptr[0] + sptr[1]) / 2.0; + sptr += 2; + } + else if (hs == 3) + for (x = 0; x < w; ++x) + { + // use kernel [1 2 1]^T / 4 + *rptr++ = (sptr[0] + sptr[1] * 2.0 + sptr[2]) / 4.0; + sptr += 3; + } + else // general case with hs >= 4 + for (x = 0; x < w; ++x) + { + // top most point - use kernel [10 10 5 1]^T / 26 + *rptr++ = ((sptr[0] + sptr[1]) * 10.0 + + sptr[2] * 5.0 + sptr[3]) / 26.0; + //++sptr; + + // general case + for (y = 0; y < (hs - 5); y += 2) + { + // use kernel [1 5 10 10 5 1]^T / 32 + *rptr++ = ((sptr[1] + sptr[4]) * 5.0 + + (sptr[2] + sptr[3]) * 10.0 + + (sptr[0] + sptr[5])) / 32.0; + sptr += 2; + } + + // find out how to treat the bottom most point + if (y == (hs - 5)) + { + // use kernel [1 5 10 10 5]^T / 31 + *rptr++ = ((sptr[1] + sptr[4]) * 5.0 + + (sptr[2] + sptr[3]) * 10.0 + + sptr[0]) / 31.0; + sptr += 5; + } + else + { + // use kernel [1 5 10 10]^T / 26 + *rptr++ = ( sptr[0] + sptr[1] * 5.0 + + (sptr[2] + sptr[3]) * 10.0) / 26.0; + sptr += 4; + } + } +} + +// ###################################################################### +// kernel: 1 5 10 10 5 1 +void lowPass6xDecX(float* sptr, float* rptr, int ws, int h) +{ + int x,y; + const int h2 = h * 2, h3 = h * 3, h4 = h * 4, h5 = h * 5; + int wr = ws / 2; + if (wr == 0) wr = 1; + + /* if (ws <= 1) + result = src; + else */ + if (ws == 2) + for (y = 0; y < h; ++y) + { + // use kernel [1 1] / 2 + *rptr++ = (sptr[0] + sptr[h]) / 2.0; + ++sptr; + } + else if (ws == 3) + for (y = 0; y < h; ++y) + { + // use kernel [1 2 1] / 4 + *rptr++ = (sptr[0] + sptr[h] * 2.0 + sptr[h2]) / 4.0; + ++sptr; + } + else // general case for ws >= 4 + { + // left most point - use kernel [10 10 5 1] / 26 + for (y = 0; y < h; ++y) + { + *rptr++ = ((sptr[0] + sptr[h]) * 10.0 + + sptr[h2] * 5.0 + sptr[h3]) / 26.0; + ++sptr; + } + sptr -= h; + + // general case + for (x = 0; x < (ws - 5); x += 2) + { + for (y = 0; y < h; ++y) + { + // use kernel [1 5 10 10 5 1] / 32 + *rptr++ = ((sptr[h] + sptr[h4]) * 5.0 + + (sptr[h2] + sptr[h3]) * 10.0 + + (sptr[0] + sptr[h5])) / 32.0; + ++sptr; + } + sptr += h; + } + + // find out how to treat the right most point + if (x == (ws - 5)) + for (y = 0; y < h; ++y) + { + // use kernel [1 5 10 10 5] / 31 + *rptr++ = ((sptr[h] + sptr[h4]) * 5.0 + + (sptr[h2] + sptr[h3]) * 10.0 + + sptr[0]) / 31.0; + ++sptr; + } + else + for (y = 0; y < h; ++y) + { + // use kernel [1 5 10 10] / 26 + *rptr++ = ( sptr[0] + sptr[h] * 5.0 + + (sptr[h2] + sptr[h3]) * 10.0) / 26.0; + ++sptr; + } + } +} diff --git a/gbvs/saltoolbox/mySubsample.mexa64 b/gbvs/saltoolbox/mySubsample.mexa64 new file mode 100755 index 0000000..aa3254a Binary files /dev/null and b/gbvs/saltoolbox/mySubsample.mexa64 differ diff --git a/gbvs/saltoolbox/mySubsample.mexglx b/gbvs/saltoolbox/mySubsample.mexglx new file mode 100755 index 0000000..bd9a78e Binary files /dev/null and b/gbvs/saltoolbox/mySubsample.mexglx differ diff --git a/gbvs/saltoolbox/mySubsample.mexmaci b/gbvs/saltoolbox/mySubsample.mexmaci new file mode 100755 index 0000000..e6b8ac2 Binary files /dev/null and b/gbvs/saltoolbox/mySubsample.mexmaci differ diff --git a/gbvs/saltoolbox/mySubsample.mexmaci64 b/gbvs/saltoolbox/mySubsample.mexmaci64 new file mode 100755 index 0000000..06973cd Binary files /dev/null and b/gbvs/saltoolbox/mySubsample.mexmaci64 differ diff --git a/gbvs/saltoolbox/mySubsample.mexw32 b/gbvs/saltoolbox/mySubsample.mexw32 new file mode 100755 index 0000000..c56bc73 Binary files /dev/null and b/gbvs/saltoolbox/mySubsample.mexw32 differ diff --git a/gbvs/saltoolbox/mySubsample.mexw64 b/gbvs/saltoolbox/mySubsample.mexw64 new file mode 100755 index 0000000..af8d735 Binary files /dev/null and b/gbvs/saltoolbox/mySubsample.mexw64 differ diff --git a/gbvs/saltoolbox/safeDivideGBVS.m b/gbvs/saltoolbox/safeDivideGBVS.m new file mode 100755 index 0000000..ba68863 --- /dev/null +++ b/gbvs/saltoolbox/safeDivideGBVS.m @@ -0,0 +1,17 @@ +function result = safeDivideGBVS(arg1,arg2) +% safeDivide - divides two arrays, checking for 0/0. +% +% result = safeDivide(arg1,arg2) +% returns arg1./arg2, where 0/0 is assumed to be 0 instead of NaN. + +% This file is part of the SaliencyToolbox - Copyright (C) 2006 +% by Dirk Walther and the California Institute of Technology. +% The Saliency Toolbox is released under the GNU General Public +% License. See the enclosed COPYRIGHT document for details. +% For more information about this project see: +% http://www.saliencytoolbox.net + +ze = (arg2 == 0); +arg2(ze) = 1; +result = arg1./arg2; +result(ze) = 0; diff --git a/gbvs/samplepics/1.jpg b/gbvs/samplepics/1.jpg new file mode 100755 index 0000000..8dc0bd7 Binary files /dev/null and b/gbvs/samplepics/1.jpg differ diff --git a/gbvs/samplepics/2.jpg b/gbvs/samplepics/2.jpg new file mode 100755 index 0000000..9637482 Binary files /dev/null and b/gbvs/samplepics/2.jpg differ diff --git a/gbvs/samplepics/3.jpg b/gbvs/samplepics/3.jpg new file mode 100755 index 0000000..d271324 Binary files /dev/null and b/gbvs/samplepics/3.jpg differ diff --git a/gbvs/samplepics/4.jpg b/gbvs/samplepics/4.jpg new file mode 100755 index 0000000..61205b4 Binary files /dev/null and b/gbvs/samplepics/4.jpg differ diff --git a/gbvs/samplepics/5.jpg b/gbvs/samplepics/5.jpg new file mode 100755 index 0000000..95dd2b4 Binary files /dev/null and b/gbvs/samplepics/5.jpg differ diff --git a/gbvs/samplepics/seq/085.jpg b/gbvs/samplepics/seq/085.jpg new file mode 100755 index 0000000..e2cff42 Binary files /dev/null and b/gbvs/samplepics/seq/085.jpg differ diff --git a/gbvs/samplepics/seq/086.jpg b/gbvs/samplepics/seq/086.jpg new file mode 100755 index 0000000..f68f61c Binary files /dev/null and b/gbvs/samplepics/seq/086.jpg differ diff --git a/gbvs/samplepics/seq/087.jpg b/gbvs/samplepics/seq/087.jpg new file mode 100755 index 0000000..e5dd52e Binary files /dev/null and b/gbvs/samplepics/seq/087.jpg differ diff --git a/gbvs/samplepics/seq/088.jpg b/gbvs/samplepics/seq/088.jpg new file mode 100755 index 0000000..31fbd0c Binary files /dev/null and b/gbvs/samplepics/seq/088.jpg differ diff --git a/gbvs/util/areaROC.m b/gbvs/util/areaROC.m new file mode 100755 index 0000000..d139e77 --- /dev/null +++ b/gbvs/util/areaROC.m @@ -0,0 +1,13 @@ +function [A,tmp] = areaROC( p ) +tmp = -1; +p = getBestRows(p); +xy = sortrows([p(:,2) p(:,1)]); + +x = xy(:,1); +y = xy(:,2); + +x = [ 0 ; x ; 1 ]; +y = [ 0 ; y ; 1 ]; + +A = trapz( x , y ); + diff --git a/gbvs/util/featureChannels/C_color.m b/gbvs/util/featureChannels/C_color.m new file mode 100755 index 0000000..3eeb1b2 --- /dev/null +++ b/gbvs/util/featureChannels/C_color.m @@ -0,0 +1,17 @@ +function out = C_color( fparam, img , imgR, imgG, imgB, typeidx ) + +if ( nargin == 1 ) + + out.weight = fparam.colorWeight; + + out.numtypes = 2; + out.descriptions{1} = 'Blue-Yellow'; + out.descriptions{2} = 'Red-Green'; + +else + if ( typeidx ) == 1 + out.map = safeDivideGBVS( abs(imgB-min(imgR,imgG)) , img ); + else + out.map = safeDivideGBVS( abs(imgR-imgG) , img ); + end +end diff --git a/gbvs/util/featureChannels/D_dklcolor.m b/gbvs/util/featureChannels/D_dklcolor.m new file mode 100755 index 0000000..7737c1f --- /dev/null +++ b/gbvs/util/featureChannels/D_dklcolor.m @@ -0,0 +1,33 @@ +function out = D_dklcolor( fparam, img , imgR, imgG, imgB, typeidx ) + +% CHROMATIC MECHANISMS IN LATERAL +% GENICULATE NUCLEUS OF MACAQUE +% BY A. M. DERRINGTON, J. KRAUSKOPF AND P. LENNIE + +% from the abstract: +% (a) an axis along +% which only luminance varies, without change in chromaticity, (b) a 'constant B' axis +% along which chromaticity varies without changing the excitation of blue-sensitive (B) +% cones, (c) a 'constant R & G' axis along which chromaticity varies without change +% in the excitation of red-sensitive (R) or green-sensitive (G) cones + +if ( nargin == 1 ) + out.weight = fparam.dklcolorWeight; + out.numtypes = 3; + out.descriptions{1} = 'DKL Luminosity Channel'; + out.descriptions{2} = 'DKL Color Channel 1'; + out.descriptions{3} = 'DKL Color Channel 2'; +else + rgb = repmat( imgR , [ 1 1 3 ] ); + rgb(:,:,2) = imgG; + rgb(:,:,3) = imgB; + dkl = rgb2dkl( rgb ); + + if ( typeidx == 1 ) + out.map = dkl(:,:,1); + elseif ( typeidx == 2 ) + out.map = dkl(:,:,2); + elseif ( typeidx == 3 ) + out.map = dkl(:,:,3); + end +end diff --git a/gbvs/util/featureChannels/F_flicker.m b/gbvs/util/featureChannels/F_flicker.m new file mode 100755 index 0000000..dc07bd2 --- /dev/null +++ b/gbvs/util/featureChannels/F_flicker.m @@ -0,0 +1,9 @@ +function out = F_flicker( param, img , img_prev, prev_img_shift , ti ) + +if ( nargin == 1 ) + out.weight = param.flickerWeight; + out.numtypes = 1; + out.descriptions{1} = 'Flicker'; +else + out.map = abs( img - img_prev ); +end diff --git a/gbvs/util/featureChannels/I_intensity.m b/gbvs/util/featureChannels/I_intensity.m new file mode 100755 index 0000000..79ed3e5 --- /dev/null +++ b/gbvs/util/featureChannels/I_intensity.m @@ -0,0 +1,9 @@ +function out = I_intensity( fparam , img , imgR, imgG, imgB, typeidx ) + +if ( nargin == 1) + out.weight = fparam.intensityWeight; + out.numtypes = 1; + out.descriptions{1} = 'Intensity'; +else + out.map = img; +end diff --git a/gbvs/util/featureChannels/M_motion.m b/gbvs/util/featureChannels/M_motion.m new file mode 100755 index 0000000..6930e7b --- /dev/null +++ b/gbvs/util/featureChannels/M_motion.m @@ -0,0 +1,18 @@ +function out = M_motion( param, img , img_prev, prev_img_shift , ti ) + +if ( nargin == 1 ) + out.weight = param.motionWeight; + out.numtypes = length( param.motionAngles ); + for i = 1 : length( param.motionAngles ), + out.descriptions{i} = sprintf('Motion Direction %g',param.motionAngles(i)); + end +else + out.imgShift = shiftImage( img , param.motionAngles(ti) ); + out.map = abs( img .* prev_img_shift - img_prev .* out.imgShift ); + + % this rule comes from: + % http://ilab.usc.edu/publications/doc/Itti_etal03spienn.pdf + % "Values smaller than 3.0 are set to zero" + % note: this doens't seem to work ? (9/4/09) + % out.map( out.map < 3/255 ) = 0; +end diff --git a/gbvs/util/featureChannels/O_orientation.m b/gbvs/util/featureChannels/O_orientation.m new file mode 100755 index 0000000..f8d6306 --- /dev/null +++ b/gbvs/util/featureChannels/O_orientation.m @@ -0,0 +1,16 @@ +function out = O_orientation( fparam , img , imgR, imgG, imgB, typeidx ) + +if ( nargin == 1 ) + out.weight = fparam.orientationWeight; + out.numtypes = length( fparam.gaborFilters ); + for i = 1 : length( fparam.gaborFilters ), + out.descriptions{i} = sprintf('Gabor Orientation %g',fparam.gaborangles(i)); + end +else + gaborFilters = fparam.gaborFilters; + j = typeidx; + f0 = myconv2(img,gaborFilters{j}.g0); + f90 = myconv2(img,gaborFilters{j}.g90); + out.map = abs(f0) + abs(f90); + out.map = attenuateBordersGBVS(out.map,13); +end diff --git a/gbvs/util/featureChannels/R_contrast.m b/gbvs/util/featureChannels/R_contrast.m new file mode 100755 index 0000000..18318ef --- /dev/null +++ b/gbvs/util/featureChannels/R_contrast.m @@ -0,0 +1,13 @@ +function out = R_contrast( fparam , img , imgR, imgG, imgB, typeidx ) + +if ( nargin == 1 ) + + out.weight = fparam.contrastWeight; + out.numtypes = 1; + out.descriptions{1} = 'Intensity Contrast'; + +else + + out.map = myContrast( img , round(size(img,1) * fparam.contrastwidth) ); + +end diff --git a/gbvs/util/getBestRows.m b/gbvs/util/getBestRows.m new file mode 100755 index 0000000..54791e5 --- /dev/null +++ b/gbvs/util/getBestRows.m @@ -0,0 +1,11 @@ +function pnew = getBestRows( p ) + +% given collection of p = [ a b ; a1 b1 ; a2 b2 .. ] +% trims out instances where bi = bj , choosing row with maximum a. + +bs = unique(p(:,2)); +Nbs = length(bs); +pnew = zeros(Nbs,2); +for i=1:Nbs, + pnew(i,:) = [ max(p(p(:,2)==bs(i),1)) bs(i) ]; +end \ No newline at end of file diff --git a/gbvs/util/getFeatureMaps.m b/gbvs/util/getFeatureMaps.m new file mode 100755 index 0000000..c77555f --- /dev/null +++ b/gbvs/util/getFeatureMaps.m @@ -0,0 +1,135 @@ +function [rawfeatmaps, motionInfo] = getFeatureMaps( img , param , prevMotionInfo ) + +% +% this computes feature maps for each cannnel in featureChannels/ +% + +load mypath; + +%%%% +%%%% STEP 1 : form image pyramid and prune levels if pyramid levels get too small. +%%%% + +mymessage(param,'forming image pyramid\n'); + +levels = [ 2 : param.maxcomputelevel ]; + +is_color = (size(img,3) == 3); +imgr = []; imgg = []; imgb = []; +if ( is_color ) [imgr,imgg,imgb,imgi] = mygetrgb( img ); +else imgi = img; end + +imgL = {}; +imgL{1} = mySubsample(imgi); +imgR{1} = mySubsample(imgr); imgG{1} = mySubsample(imgg); imgB{1} = mySubsample(imgb); + +for i=levels + + imgL{i} = mySubsample( imgL{i-1} ); + if ( is_color ) + imgR{i} = mySubsample( imgR{i-1} ); + imgG{i} = mySubsample( imgG{i-1} ); + imgB{i} = mySubsample( imgB{i-1} ); + else + imgR{i} = []; imgG{i} = []; imgB{i} = []; + end + if ( (size(imgL{i},1) < 3) | (size(imgL{i},2) < 3 ) ) + mymessage(param,'reached minimum size at level = %d. cutting off additional levels\n', i); + levels = [ 2 : i ]; + param.maxcomputelevel = i; + break; + end + +end + +%%% update previous frame estimate based on new frame +if ( (param.flickerNewFrameWt == 1) || (isempty(prevMotionInfo) ) ) + motionInfo.imgL = imgL; +else + w = param.flickerNewFrameWt; + for i = levels, + %%% new frame gets weight flickerNewFrameWt + motionInfo.imgL = w * imgL{i} + ( 1 - w ) * prevMotionInfo.imgL{i}; + end +end + +%%% +%%% STEP 2 : compute feature maps +%%% + +mymessage(param,'computing feature maps...\n'); + +rawfeatmaps = {}; + +%%% get channel functions in featureChannels/directory + +channel_files = dir( [pathroot '/util/featureChannels/*.m'] ); + +motionInfo.imgShifts = {}; + +for ci = 1 : length(channel_files) + + %%% parse the channel letter and name from filename + parts = regexp( channel_files(ci).name , '^(?\w)_(?.*?)\.m$' , 'names'); + if ( isempty(parts) ), continue; end % invalid channel file name + + channelLetter = parts.letter; + channelName = parts.rest; + channelfunc = str2func(sprintf('%s_%s',channelLetter,channelName)); + useChannel = sum(param.channels==channelLetter) > 0; + + if ( ((channelLetter == 'C') || (channelLetter=='D')) && useChannel && (~is_color) ) + mymessage(param,'oops! cannot compute color channel on black and white image. skipping this channel\n'); + continue; + elseif (useChannel) + + mymessage(param,'computing feature maps of type "%s" ... \n', channelName); + + obj = {}; + obj.info = channelfunc(param); + obj.description = channelName; + + obj.maps = {}; + obj.maps.val = {}; + + %%% call the channelfunc() for each desired image resolution (level in pyramid) + %%% and for each type index for this channel. + + for ti = 1 : obj.info.numtypes + obj.maps.val{ti} = {}; + mymessage(param,'..pyramid levels: '); + for lev = levels, + mymessage(param,'%d (%d x %d)', lev, size(imgL{lev},1), size(imgL{lev},2)); + if ( (channelLetter == 'F') || (channelLetter == 'M') ) + if ( ~isempty(prevMotionInfo) ) + prev_img = prevMotionInfo.imgL{lev}; + else + prev_img = imgL{lev}; + end + + if ( ~isempty(prevMotionInfo) && isfield(prevMotionInfo,'imgShifts') && (channelLetter == 'M') ) + prev_img_shift = prevMotionInfo.imgShifts{ti}{lev}; + else + prev_img_shift = 0; + end + + map = channelfunc(param,imgL{lev},prev_img,prev_img_shift,ti); + if (isfield(map,'imgShift')) + motionInfo.imgShifts{ti}{lev} = map.imgShift; + end + else + map = channelfunc(param,imgL{lev},imgR{lev},imgG{lev},imgB{lev},ti); + end + obj.maps.origval{ti}{lev} = map.map; + map = imresize( map.map , param.salmapsize , 'bicubic' ); + obj.maps.val{ti}{lev} = map; + end + mymessage(param,'\n'); + end + + %%% save output to rawfeatmaps structure + eval( sprintf('rawfeatmaps.%s = obj;', channelName) ); + + end +end + diff --git a/gbvs/util/getIntelligentThresholds.m b/gbvs/util/getIntelligentThresholds.m new file mode 100755 index 0000000..d2f8b26 --- /dev/null +++ b/gbvs/util/getIntelligentThresholds.m @@ -0,0 +1,8 @@ +function threshs = getIntelligentThresholds( vals ); + +threshs = unique(vals) - 1e-16; + + + + + diff --git a/gbvs/util/heatmap_overlay.m b/gbvs/util/heatmap_overlay.m new file mode 100755 index 0000000..d360697 --- /dev/null +++ b/gbvs/util/heatmap_overlay.m @@ -0,0 +1,29 @@ + +% img = image on which to overlay heatmap +% heatmap = the heatmap +% (optional) colorfunc .. this can be 'jet' , or 'hot' , or 'flag' + +function omap = heatmap_overlay( img , heatmap, colorfun ) + +if ( strcmp(class(img),'char') == 1 ) img = imread(img); end +if ( strcmp(class(img),'uint8') == 1 ) img = double(img)/255; end + +szh = size(heatmap); +szi = size(img); + +if ( (szh(1)~=szi(1)) | (szh(2)~=szi(2)) ) + heatmap = imresize( heatmap , [ szi(1) szi(2) ] , 'bicubic' ); +end + +if ( size(img,3) == 1 ) + img = repmat(img,[1 1 3]); +end + +if ( nargin == 2 ) + colorfun = 'jet'; +end +colorfunc = eval(sprintf('%s(50)',colorfun)); + +heatmap = double(heatmap) / max(heatmap(:)); +omap = 0.8*(1-repmat(heatmap.^0.8,[1 1 3])).*double(img)/max(double(img(:))) + repmat(heatmap.^0.8,[1 1 3]).* shiftdim(reshape( interp2(1:3,1:50,colorfunc,1:3,1+49*reshape( heatmap , [ prod(size(heatmap)) 1 ] ))',[ 3 size(heatmap) ]),1); +omap = real(omap); \ No newline at end of file diff --git a/gbvs/util/invCenterBias.mat b/gbvs/util/invCenterBias.mat new file mode 100755 index 0000000..e2c1c69 Binary files /dev/null and b/gbvs/util/invCenterBias.mat differ diff --git a/gbvs/util/linearmap.m b/gbvs/util/linearmap.m new file mode 100755 index 0000000..ff81b62 --- /dev/null +++ b/gbvs/util/linearmap.m @@ -0,0 +1,4 @@ +function lmap = linearmap(map) +[n,m] = size(map); +lmap = reshape(map, [1 n*m]); + diff --git a/gbvs/util/makeFixationMask.m b/gbvs/util/makeFixationMask.m new file mode 100755 index 0000000..7513a5d --- /dev/null +++ b/gbvs/util/makeFixationMask.m @@ -0,0 +1,31 @@ + +function mask = makeFixationMask( X , Y , origimgsize , salmapsize ) + +% +% this maps (X,Y) fixation coordinates to fixation mask +% +% given fixation coordinates X and Y in original image coordinates, +% produces mask of same size salmapsize where each location contains +% an integer count of the fixations lying at that location +% + +if ( length(X) ~= length(Y) ) + fprintf(2,'makeFixationMask Error: number of X and Y coordinates should be the same!\n'); + mask = []; + return; +end + +scale = salmapsize(1) / origimgsize(1); + +X = round(X * scale); +Y = round(Y * scale); + +X(X<1) = 1; +X(X>salmapsize(2)) = salmapsize(2); +Y(Y<1) = 1; +Y(Y>salmapsize(1)) = salmapsize(1); + +mask = zeros( salmapsize ); +for i = 1 : length(X) + mask( Y(i) , X(i) ) = mask( Y(i) , X(i) ) + 1; +end \ No newline at end of file diff --git a/gbvs/util/myContrast.cc b/gbvs/util/myContrast.cc new file mode 100755 index 0000000..c0f725e --- /dev/null +++ b/gbvs/util/myContrast.cc @@ -0,0 +1,82 @@ +#include +#include +#include +#include +#include +#include + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + + //---Inside mexFunction--- + + //Declarations + + mxArray *xData, *sData; + double *xValues, *outArray, *sValues; + + int i,j,k,z; + int rowLen, colLen; + + int M; + int Mo2; + double sumPix; + double sumPixsq; + double gamma = 2.0; + double StoDelta = 1.0; + double weight; + double delta = 1.0; + double p; + double var; + double ni; + int kv; + int zv; + + // get first argument x + xData = (mxArray*)prhs[0]; + xValues = mxGetPr(xData); + rowLen = mxGetN(xData); + colLen = mxGetM(xData); + + // get second argument sigma + sData = (mxArray*)prhs[1]; + sValues = mxGetPr(sData); + M = (int)sValues[0]; + Mo2 = (int)(M/2); + + //Allocate memory and assign output pointer + plhs[0] = mxCreateDoubleMatrix(colLen, rowLen, mxREAL); //mxReal is our data-type + + //Get a pointer to the data space in our newly allocated memory + outArray = mxGetPr(plhs[0]); + + //Copy matrix while multiplying each point by 2 + for(i=0;i= rowLen) kv = kv - rowLen; + if ( zv < 0 ) zv = colLen + zv; + else if ( zv >= colLen ) zv = zv - colLen; */ + if ( kv < 0 || kv >= rowLen || zv < 0 || zv >= colLen ) continue; + p = xValues[(kv*colLen)+zv]; + sumPix += p; + sumPixsq += p*p; + ni++; + } + } + var = 0; + if ( ni != 0 ) { + sumPix /= ni; + var = sumPixsq - ni * sumPix * sumPix; + var /= ni; + } + outArray[(i*colLen)+j] = var; + } + } + return; +} diff --git a/gbvs/util/myContrast.mexa64 b/gbvs/util/myContrast.mexa64 new file mode 100755 index 0000000..dbfc88c Binary files /dev/null and b/gbvs/util/myContrast.mexa64 differ diff --git a/gbvs/util/myContrast.mexglx b/gbvs/util/myContrast.mexglx new file mode 100755 index 0000000..3dbaa09 Binary files /dev/null and b/gbvs/util/myContrast.mexglx differ diff --git a/gbvs/util/myContrast.mexmaci b/gbvs/util/myContrast.mexmaci new file mode 100755 index 0000000..042db70 Binary files /dev/null and b/gbvs/util/myContrast.mexmaci differ diff --git a/gbvs/util/myContrast.mexmaci64 b/gbvs/util/myContrast.mexmaci64 new file mode 100755 index 0000000..c790879 Binary files /dev/null and b/gbvs/util/myContrast.mexmaci64 differ diff --git a/gbvs/util/myContrast.mexw32 b/gbvs/util/myContrast.mexw32 new file mode 100755 index 0000000..877fe28 Binary files /dev/null and b/gbvs/util/myContrast.mexw32 differ diff --git a/gbvs/util/myContrast.mexw64 b/gbvs/util/myContrast.mexw64 new file mode 100755 index 0000000..d4bf253 Binary files /dev/null and b/gbvs/util/myContrast.mexw64 differ diff --git a/gbvs/util/mycombnk.m b/gbvs/util/mycombnk.m new file mode 100755 index 0000000..5836287 --- /dev/null +++ b/gbvs/util/mycombnk.m @@ -0,0 +1,30 @@ +% so that stats toolbox is not used +function cmbs = mycombnk( nums , k ) + +N = length(nums); +T = N^k; +cmbs = zeros( T , k ); + +for j=T:-1:1 + n = j; + for jj=1:k + b = mod(n,N); + n = n - b; + n = n / N; + cmbs( j , k-jj+1 ) = nums(b+1); + cmbs( j , : ) = sort( cmbs(j,:) ); + end + is_used = zeros( N , 1 ); + for jj=1:k + if (is_used( cmbs(j,jj))) + cmbs(j,:) = -1 * ones(1,k); + break; + else + is_used( cmbs(j,jj) ) = 1; + end + end +end + + +cmbs = unique(cmbs,'rows'); +cmbs = cmbs(2:end,:); \ No newline at end of file diff --git a/gbvs/util/myconv2.m b/gbvs/util/myconv2.m new file mode 100755 index 0000000..02a5c4d --- /dev/null +++ b/gbvs/util/myconv2.m @@ -0,0 +1,12 @@ +function c = myconv2(a,b) + +% +% conv2 with 'same' and repeating boundary condition +% + +vpad = ceil(( size(b,1) - 1 ) / 2); +hpad = ceil(( size(b,2) - 1 ) / 2); +ap = padImage( a , vpad , hpad ); +cp = conv2( ap , b , 'same' ); +c = cp( vpad + 1 : vpad + size(a,1) , hpad + 1 : hpad + size(a,2) ); + diff --git a/gbvs/util/mygausskernel.m b/gbvs/util/mygausskernel.m new file mode 100755 index 0000000..c0dd9f2 --- /dev/null +++ b/gbvs/util/mygausskernel.m @@ -0,0 +1,11 @@ +function k = mygausskernel( std , nstds ) + +maxi = round(std * nstds); +a = [ normpdf(0,0,std) zeros(1,maxi) ]; + +for i = 1 : maxi + a(1+i) = normpdf( i , 0 , std ); +end + +k = [ a(end:-1:2) a ]; +k = k / sum(k); diff --git a/gbvs/util/mygetrgb.m b/gbvs/util/mygetrgb.m new file mode 100755 index 0000000..cfe6f26 --- /dev/null +++ b/gbvs/util/mygetrgb.m @@ -0,0 +1,6 @@ +function [r,g,b,ii] = mygetrgb( img ) + + r = img(:,:,1); + g = img(:,:,2); + b = img(:,:,3); + ii = max(max(r,g),b); diff --git a/gbvs/util/mymessage.m b/gbvs/util/mymessage.m new file mode 100755 index 0000000..afa44a8 --- /dev/null +++ b/gbvs/util/mymessage.m @@ -0,0 +1,19 @@ +function mymessage(param,s,varargin) +if (param.verbose == 1 ) + s = [ '\t' s ]; + if ( strcmp(param.verboseout,'screen') == 1 ) + if ( nargin == 2 ) + fprintf(s); + else + fprintf(s,varargin{:}); + end + else + fido = fopen(param.verboseout,'a'); + if ( nargin == 2 ) + fprintf(fido,s); + else + fprintf(fido,s,varargin{:}); + end + fclose(fido); + end +end \ No newline at end of file diff --git a/gbvs/util/mypath.mat b/gbvs/util/mypath.mat new file mode 100755 index 0000000..da4b0db Binary files /dev/null and b/gbvs/util/mypath.mat differ diff --git a/gbvs/util/padImage.m b/gbvs/util/padImage.m new file mode 100755 index 0000000..f25ede2 --- /dev/null +++ b/gbvs/util/padImage.m @@ -0,0 +1,20 @@ +function b = padImage( a , vpad , hpad ) + +if ( nargin == 2 ) + hpad = vpad; +end + +u = repmat( a(1,:) , [ vpad 1 ] ); +b = repmat( a(end,:) , [ vpad 1 ] ); + +l = repmat( a(:,1) , [ 1 hpad ] ); +r = repmat( a(:,end) , [ 1 hpad ] ); + +ul = repmat( a(1,1) , [ vpad hpad ] ); +ur = repmat( a(1,end) , [ vpad hpad ] ); +bl = repmat( a(end,1) , [ vpad hpad ] ); +br = repmat( a(end,end) , [ vpad hpad ] ); + +b = [ ul u ur + l a r + bl b br ]; diff --git a/gbvs/util/padImageOld.m b/gbvs/util/padImageOld.m new file mode 100755 index 0000000..385e163 --- /dev/null +++ b/gbvs/util/padImageOld.m @@ -0,0 +1,33 @@ +function b = padImageOld( a , vpad , hpad ) + +if ( nargin == 2 ) + hpad = vpad; +end + +if ( (size(a,1) > (2*vpad)) && (size(a,2) > (2*hpad)) ) + u = a(vpad:-1:1,:); + b = a(end:-1:end-vpad+1,:); + + l = a(:,hpad:-1:1); + r = a(:,end:-1:end-hpad+1); + + ul = a(vpad:-1:1,hpad:-1:1); + ur = a(vpad:-1:1,end:-1:end-hpad+1); + bl = a(end:-1:end-vpad+1,hpad:-1:1); + br = a(end:-1:end-vpad+1,end:-1:end-hpad+1); +else + u = repmat( a(1,:) , [ vpad 1 ] ); + b = repmat( a(end,:) , [ vpad 1 ] ); + + l = repmat( a(:,1) , [ 1 hpad ] ); + r = repmat( a(:,end) , [ 1 hpad ] ); + + ul = repmat( a(1,1) , [ vpad hpad ] ); + ur = repmat( a(1,end) , [ vpad hpad ] ); + bl = repmat( a(end,1) , [ vpad hpad ] ); + br = repmat( a(end,end) , [ vpad hpad ] ); +end + +b = [ ul u ur + l a r + bl b br ]; diff --git a/gbvs/util/rankimg.m b/gbvs/util/rankimg.m new file mode 100755 index 0000000..e23815f --- /dev/null +++ b/gbvs/util/rankimg.m @@ -0,0 +1,11 @@ +function rimg = rankimg(img) + +img = uint8(mat2gray(img)*255); + +rimg = zeros(size(img)); + +img = img(:); + +for i = 0 : 255 + rimg( img == i ) = mean( img < i ); +end \ No newline at end of file diff --git a/gbvs/util/rgb2dkl.m b/gbvs/util/rgb2dkl.m new file mode 100755 index 0000000..867a995 --- /dev/null +++ b/gbvs/util/rgb2dkl.m @@ -0,0 +1,312 @@ + +function dkl = rgb2dkl(rgb) + + + +sz = size(rgb); +im = shiftdim(rgb,2); +im = reshape(im,[3 prod(sz(1:2))]); +im = im'; +im = rgb2dkl_v(im); +im = im'; +im = reshape(im,[3 sz(1:2)]); +dkl = shiftdim(im,1); + +function dkl = rgb2dkl_v(rgb) + +% bunch of constants used for RGB -> DKL conversion: +lut_rgb = [ + 0.024935, 0.0076954, 0.042291, + 0.024974, 0.0077395, 0.042346, + 0.025013, 0.0077836, 0.042401, + 0.025052, 0.0078277, 0.042456, + 0.025091, 0.0078717, 0.042511, + 0.02513, 0.0079158, 0.042566, + 0.025234, 0.007992, 0.042621, + 0.025338, 0.0080681, 0.042676, + 0.025442, 0.0081443, 0.042731, + 0.025545, 0.0082204, 0.042786, + 0.025649, 0.0082966, 0.042841, + 0.025747, 0.0084168, 0.042952, + 0.025844, 0.0085371, 0.043062, + 0.025942, 0.0086573, 0.043172, + 0.026039, 0.0087776, 0.043282, + 0.026136, 0.0088978, 0.043392, + 0.026234, 0.0090581, 0.043502, + 0.026331, 0.0092184, 0.043612, + 0.026429, 0.0093788, 0.043722, + 0.026526, 0.0095391, 0.043833, + 0.026623, 0.0096994, 0.043943, + 0.026818, 0.0099198, 0.044141, + 0.027013, 0.01014, 0.044339, + 0.027208, 0.010361, 0.044537, + 0.027403, 0.010581, 0.044736, + 0.027597, 0.010802, 0.044934, + 0.027857, 0.010994, 0.04522, + 0.028117, 0.011186, 0.045507, + 0.028377, 0.011379, 0.045793, + 0.028636, 0.011571, 0.046079, + 0.028896, 0.011764, 0.046366, + 0.029104, 0.012068, 0.046652, + 0.029312, 0.012373, 0.046938, + 0.029519, 0.012677, 0.047225, + 0.029727, 0.012982, 0.047511, + 0.029935, 0.013287, 0.047797, + 0.030273, 0.013663, 0.048326, + 0.03061, 0.01404, 0.048855, + 0.030948, 0.014417, 0.049383, + 0.031286, 0.014794, 0.049912, + 0.031623, 0.01517, 0.050441, + 0.032156, 0.015707, 0.051035, + 0.032688, 0.016244, 0.05163, + 0.033221, 0.016782, 0.052225, + 0.033753, 0.017319, 0.052819, + 0.034286, 0.017856, 0.053414, + 0.034961, 0.018693, 0.054493, + 0.035636, 0.019531, 0.055573, + 0.036312, 0.020369, 0.056652, + 0.036987, 0.021206, 0.057731, + 0.037662, 0.022044, 0.058811, + 0.038623, 0.023246, 0.060044, + 0.039584, 0.024449, 0.061278, + 0.040545, 0.025651, 0.062511, + 0.041506, 0.026854, 0.063744, + 0.042468, 0.028056, 0.064978, + 0.043857, 0.029659, 0.066806, + 0.045247, 0.031263, 0.068634, + 0.046636, 0.032866, 0.070463, + 0.048026, 0.034469, 0.072291, + 0.049416, 0.036072, 0.074119, + 0.051221, 0.038156, 0.076476, + 0.053026, 0.04024, 0.078833, + 0.054831, 0.042325, 0.081189, + 0.056636, 0.044409, 0.083546, + 0.058442, 0.046493, 0.085903, + 0.06039, 0.048737, 0.087996, + 0.062338, 0.050982, 0.090088, + 0.064286, 0.053226, 0.092181, + 0.066234, 0.055471, 0.094273, + 0.068182, 0.057715, 0.096366, + 0.070519, 0.06012, 0.098921, + 0.072857, 0.062525, 0.10148, + 0.075195, 0.06493, 0.10403, + 0.077532, 0.067335, 0.10659, + 0.07987, 0.069739, 0.10914, + 0.082208, 0.072345, 0.11176, + 0.084545, 0.07495, 0.11438, + 0.086883, 0.077555, 0.117, + 0.089221, 0.08016, 0.11963, + 0.091558, 0.082766, 0.12225, + 0.094026, 0.085611, 0.12533, + 0.096494, 0.088457, 0.12841, + 0.098961, 0.091303, 0.1315, + 0.10143, 0.094148, 0.13458, + 0.1039, 0.096994, 0.13767, + 0.10688, 0.10028, 0.14119, + 0.10987, 0.10357, 0.14471, + 0.11286, 0.10685, 0.14824, + 0.11584, 0.11014, 0.15176, + 0.11883, 0.11343, 0.15529, + 0.12208, 0.11695, 0.15903, + 0.12532, 0.12048, 0.16278, + 0.12857, 0.12401, 0.16652, + 0.13182, 0.12754, 0.17026, + 0.13506, 0.13106, 0.17401, + 0.1387, 0.13499, 0.17819, + 0.14234, 0.13892, 0.18238, + 0.14597, 0.14285, 0.18656, + 0.14961, 0.14677, 0.19075, + 0.15325, 0.1507, 0.19493, + 0.15727, 0.15519, 0.19956, + 0.1613, 0.15968, 0.20419, + 0.16532, 0.16417, 0.20881, + 0.16935, 0.16866, 0.21344, + 0.17338, 0.17315, 0.21806, + 0.17805, 0.17796, 0.22291, + 0.18273, 0.18277, 0.22775, + 0.1874, 0.18758, 0.2326, + 0.19208, 0.19238, 0.23744, + 0.19675, 0.19719, 0.24229, + 0.20156, 0.20224, 0.24758, + 0.20636, 0.20729, 0.25286, + 0.21117, 0.21234, 0.25815, + 0.21597, 0.21739, 0.26344, + 0.22078, 0.22244, 0.26872, + 0.2261, 0.22806, 0.27423, + 0.23143, 0.23367, 0.27974, + 0.23675, 0.23928, 0.28524, + 0.24208, 0.24489, 0.29075, + 0.2474, 0.2505, 0.29626, + 0.25299, 0.25651, 0.3022, + 0.25857, 0.26253, 0.30815, + 0.26416, 0.26854, 0.3141, + 0.26974, 0.27455, 0.32004, + 0.27532, 0.28056, 0.32599, + 0.28156, 0.28697, 0.33238, + 0.28779, 0.29339, 0.33877, + 0.29403, 0.2998, 0.34515, + 0.30026, 0.30621, 0.35154, + 0.30649, 0.31263, 0.35793, + 0.3126, 0.31904, 0.36388, + 0.3187, 0.32545, 0.36982, + 0.32481, 0.33186, 0.37577, + 0.33091, 0.33828, 0.38172, + 0.33701, 0.34469, 0.38767, + 0.34325, 0.3511, 0.39361, + 0.34948, 0.35752, 0.39956, + 0.35571, 0.36393, 0.40551, + 0.36195, 0.37034, 0.41145, + 0.36818, 0.37675, 0.4174, + 0.37429, 0.38317, 0.42313, + 0.38039, 0.38958, 0.42885, + 0.38649, 0.39599, 0.43458, + 0.3926, 0.4024, 0.44031, + 0.3987, 0.40882, 0.44604, + 0.40494, 0.41523, 0.45198, + 0.41117, 0.42164, 0.45793, + 0.4174, 0.42806, 0.46388, + 0.42364, 0.43447, 0.46982, + 0.42987, 0.44088, 0.47577, + 0.43623, 0.44689, 0.48128, + 0.4426, 0.45291, 0.48678, + 0.44896, 0.45892, 0.49229, + 0.45532, 0.46493, 0.4978, + 0.46169, 0.47094, 0.5033, + 0.46792, 0.47695, 0.50837, + 0.47416, 0.48297, 0.51344, + 0.48039, 0.48898, 0.5185, + 0.48662, 0.49499, 0.52357, + 0.49286, 0.501, 0.52863, + 0.49805, 0.50701, 0.53392, + 0.50325, 0.51303, 0.53921, + 0.50844, 0.51904, 0.54449, + 0.51364, 0.52505, 0.54978, + 0.51883, 0.53106, 0.55507, + 0.52442, 0.53667, 0.55969, + 0.53, 0.54228, 0.56432, + 0.53558, 0.5479, 0.56894, + 0.54117, 0.55351, 0.57357, + 0.54675, 0.55912, 0.57819, + 0.55182, 0.56433, 0.58304, + 0.55688, 0.56954, 0.58789, + 0.56195, 0.57475, 0.59273, + 0.56701, 0.57996, 0.59758, + 0.57208, 0.58517, 0.60242, + 0.57675, 0.58998, 0.60639, + 0.58143, 0.59479, 0.61035, + 0.5861, 0.5996, 0.61432, + 0.59078, 0.60441, 0.61828, + 0.59545, 0.60922, 0.62225, + 0.60065, 0.61403, 0.62709, + 0.60584, 0.61884, 0.63194, + 0.61104, 0.62365, 0.63678, + 0.61623, 0.62846, 0.64163, + 0.62143, 0.63327, 0.64648, + 0.62584, 0.63808, 0.65088, + 0.63026, 0.64289, 0.65529, + 0.63468, 0.6477, 0.65969, + 0.63909, 0.65251, 0.6641, + 0.64351, 0.65731, 0.6685, + 0.64857, 0.66132, 0.67269, + 0.65364, 0.66533, 0.67687, + 0.6587, 0.66934, 0.68106, + 0.66377, 0.67335, 0.68524, + 0.66883, 0.67735, 0.68943, + 0.67273, 0.68136, 0.69361, + 0.67662, 0.68537, 0.6978, + 0.68052, 0.68938, 0.70198, + 0.68442, 0.69339, 0.70617, + 0.68831, 0.69739, 0.71035, + 0.69221, 0.7022, 0.7141, + 0.6961, 0.70701, 0.71784, + 0.7, 0.71182, 0.72159, + 0.7039, 0.71663, 0.72533, + 0.70779, 0.72144, 0.72907, + 0.71169, 0.72505, 0.73348, + 0.71558, 0.72866, 0.73789, + 0.71948, 0.73226, 0.74229, + 0.72338, 0.73587, 0.7467, + 0.72727, 0.73948, 0.7511, + 0.73247, 0.74349, 0.75507, + 0.73766, 0.74749, 0.75903, + 0.74286, 0.7515, 0.763, + 0.74805, 0.75551, 0.76696, + 0.75325, 0.75952, 0.77093, + 0.75714, 0.76393, 0.77599, + 0.76104, 0.76834, 0.78106, + 0.76494, 0.77275, 0.78612, + 0.76883, 0.77715, 0.79119, + 0.77273, 0.78156, 0.79626, + 0.77792, 0.78677, 0.80132, + 0.78312, 0.79198, 0.80639, + 0.78831, 0.79719, 0.81145, + 0.79351, 0.8024, 0.81652, + 0.7987, 0.80762, 0.82159, + 0.80519, 0.81283, 0.82687, + 0.81169, 0.81804, 0.83216, + 0.81818, 0.82325, 0.83744, + 0.82468, 0.82846, 0.84273, + 0.83117, 0.83367, 0.84802, + 0.83636, 0.83888, 0.85286, + 0.84156, 0.84409, 0.85771, + 0.84675, 0.8493, 0.86256, + 0.85195, 0.85451, 0.8674, + 0.85714, 0.85972, 0.87225, + 0.86364, 0.86613, 0.87819, + 0.87013, 0.87255, 0.88414, + 0.87662, 0.87896, 0.89009, + 0.88312, 0.88537, 0.89604, + 0.88961, 0.89178, 0.90198, + 0.8961, 0.8986, 0.90947, + 0.9026, 0.90541, 0.91696, + 0.90909, 0.91222, 0.92445, + 0.91558, 0.91904, 0.93194, + 0.92208, 0.92585, 0.93943, + 0.92857, 0.93307, 0.94493, + 0.93506, 0.94028, 0.95044, + 0.94156, 0.94749, 0.95595, + 0.94805, 0.95471, 0.96145, + 0.95455, 0.96192, 0.96696, + 0.96364, 0.96954, 0.97357, + 0.97273, 0.97715, 0.98018, + 0.98182, 0.98477, 0.98678, + 0.99091, 0.99238, 0.99339, + 1, 1, 1 ]; + +lms0 = [ 34.918538957799996, 19.314796676499999, 0.585610818500000 ]; + +m = [ 18.32535, 44.60077, 7.46216, 4.09544, 28.20135, 6.66066, 0.02114, 0.10325, 1.05258 ]; + +fac = 1.0 / (lms0(1) + lms0(2)); +mm = [ sqrt(3.0)*fac, sqrt(3.0)*fac, 0.0, sqrt(lms0(1)*lms0(1)+lms0(2)*lms0(2))/lms0(1)*fac, -sqrt(lms0(1)*lms0(1)+lms0(2)*lms0(2))/lms0(2)*fac, 0.0, -fac, -fac, (lms0(1) + lms0(2)) / lms0(3) * fac ]; + +% do a lookup RGB -> rgb: +if ( strcmp(class(rgb),'double') == 1 ) + rgb = ceil( rgb*255 ); + rgb(rgb<1) = 1; + rgb(rgb>256) = 256; +elseif ( strcmp(class(rgb),'uint8') == 1 ) + rgb = rgb + 1; +else + fprintf(2,'Oops! Don''t know what kind of rgb image this is! Hoping for the best...\n'); + rgb = ceil( rgb*255 ); + rgb(rgb<1) = 1; + rgb(rgb>256) = 256; +end + +aa1 = lut_rgb( rgb(:,1) , 1 ); +aa2 = lut_rgb( rgb(:,2) , 2 ); +aa3 = lut_rgb( rgb(:,3) , 3 ); + +% now convert to LMS: +lms1 = m(1) * aa1 + m(2) * aa2 + m(3) * aa3 - lms0(1); +lms2 = m(4) * aa1 + m(5) * aa2 + m(6) * aa3 - lms0(2); +lms3 = m(7) * aa1 + m(8) * aa2 + m(9) * aa3 - lms0(3); + +% finally to DKL: +dkl1 = mm(1) * lms1 + mm(2) * lms2 + mm(3) * lms3; +dkl2 = mm(4) * lms1 + mm(5) * lms2 + mm(6) * lms3; +dkl3 = mm(7) * lms1 + mm(8) * lms2 + mm(9) * lms3; + +% finally to DKLn: +dkl = [ dkl1 * 0.5774 dkl2 * 2.7525 dkl3 * 0.4526 ]; diff --git a/gbvs/util/rocSal.m b/gbvs/util/rocSal.m new file mode 100755 index 0000000..762173d --- /dev/null +++ b/gbvs/util/rocSal.m @@ -0,0 +1,70 @@ + +function a = rocSal( salmap , mask ) + +% ROC area agreement between saliency map (salmap) and fixations (mask) +% == good measure of HOW WELL salmap 'predicts' fixations +% +% - mask is the same size as salmap and +% contains number of fixations at each +% map location ( 0,1,2,..etc. ) +% +% - gives the ROC score of the following binary classification problem +% +% the set of trials is this +% {each (fixation,location)} UNION +% {each location on the map without a fixation} +% +% a true positive occurs +% when a (fixation,location) pair is above threshold +% a true negative occurs +% when a (no fixation,location) pair is below threshold +% a false negative occurs +% when a (fixation,location) pair is below threshold +% a false positive occurs +% when a (no fixation,location) pair is above threshold +% +% ROC curve plots TPR against FPR +% where TPR = TP / (TP+FN) = TP / (number of ground-truth trues) +% FPR = FP / (FP+TN) = FP / (number of ground-truth falses) +% +% so if out of 10 fixations, 9 occur at one location, which is the +% only above threshold location, the TPR is 90%, and the FPR is 0% +% + +% limit to 256 unique values +salmap = mat2gray(salmap); +if ( strcmp(class(salmap),'double') ) + % salmap = uint8(salmap * 255); + salmap = uint8(salmap * 50); +end + +t = getIntelligentThresholds( salmap(:) ); +Nt = length(t); +p = zeros(Nt,2); + +% number of (fixation,location) trials +Ntrues = sum(mask(:)); + +% number of (no fixation,location) trials +falses = (mask==0); +Nfalses = sum( falses(:) ); +if ( Nfalses == 0 ) Nfalses = 1e-6; end + +for ti = 1 : Nt + + T = t(ti); + shouldbefix = salmap >= T; + + TPm = mask .* shouldbefix; + TP = sum( TPm(:) ); + tpr = TP / Ntrues; + + FPm = (mask==0) .* shouldbefix; + FP = sum( FPm(:) ); + fpr = FP / Nfalses; + + p(ti,:) = [ tpr fpr ]; + +end + +a = areaROC(p); diff --git a/gbvs/util/rocScoreSaliencyVsFixations.m b/gbvs/util/rocScoreSaliencyVsFixations.m new file mode 100755 index 0000000..95a96bb --- /dev/null +++ b/gbvs/util/rocScoreSaliencyVsFixations.m @@ -0,0 +1,13 @@ + +function a = rocScoreSaliencyVsFixations( salmap , X , Y , origimgsize ) + +% +% outputs ROC Area-Under-Curve Score between a saliency map and fixations. +% +% salmap : a saliency map +% X : vector of X locations of fixations in original image coordinates +% Y : vector of Y locations of fixations in original image coordinates +% origimgsize : size of original image (should have same aspect ratio as saliency map) +% + +a = rocSal( salmap , makeFixationMask( X , Y , origimgsize , size(salmap) ) ); diff --git a/gbvs/util/shiftImage.m b/gbvs/util/shiftImage.m new file mode 100755 index 0000000..a30a73c --- /dev/null +++ b/gbvs/util/shiftImage.m @@ -0,0 +1,18 @@ +function imgShift = shiftImage( img , theta ) + +Worig = size(img,2); +Horig = size(img,1); + +pad = 2; +imgpad = padImage(img,pad); + +W = size(imgpad,2); +H = size(imgpad,1); +xi = repmat( [ 1 : W ] , [ H 1 ] ); +yi = repmat( [ 1 : H ]', [ 1 W ] ); + +dx = cos(theta * pi / 180 ); +dy = -sin(theta * pi / 180); +imgpadshift = interp2( xi , yi , imgpad , xi + dx , yi + dy ); + +imgShift = imgpadshift( pad + 1 : pad + Horig , pad + 1 : pad + Worig ); \ No newline at end of file diff --git a/gbvs/util/show_imgnmap.m b/gbvs/util/show_imgnmap.m new file mode 100755 index 0000000..b53b83f --- /dev/null +++ b/gbvs/util/show_imgnmap.m @@ -0,0 +1,3 @@ +function hm = show_imgnmap( img , out ) +hm = heatmap_overlay( img , out.master_map_resized ); +imshow( hm ); diff --git a/gbvs/util/show_imgnmap2.m b/gbvs/util/show_imgnmap2.m new file mode 100755 index 0000000..6887fb5 --- /dev/null +++ b/gbvs/util/show_imgnmap2.m @@ -0,0 +1,3 @@ +function show_imgnmap2( img , smap ) +smap = mat2gray( imresize(smap,[size(img,1) size(img,2)]) ); +imshow(heatmap_overlay( img , smap )); \ No newline at end of file diff --git a/tajmahal.png b/tajmahal.png new file mode 100755 index 0000000..74c585b Binary files /dev/null and b/tajmahal.png differ diff --git a/tajmahal_0.50_sns.png b/tajmahal_0.50_sns.png new file mode 100755 index 0000000..f6d5303 Binary files /dev/null and b/tajmahal_0.50_sns.png differ