Logistic/Softmax Regression

辅助函数

牛顿法介绍

 %% Logistic Regression
 close all
 clear

 %%load data
 x = load('ex4x.dat');
 y = load('ex4y.dat');

 [m, n] = size(x);

 % Add intercept term to x
 x = [ones(m, ), x];

 %%draw picture
 % find returns the indices of the
 % rows meeting the specified condition
 pos = find(y == );
 neg = find(y == );
 % Assume the features are in the 2nd and 3rd
 % columns of x
 figure('NumberTitle', 'off', 'Name', 'GD');
 plot(x(pos, ), x(pos,), '+');
 hold on;
 plot(x(neg, ), x(neg, ), 'o');

 % Define the sigmoid function
 g = inline('1 ./ (1 + exp(-z))');

 alpha = 0.001;
 theta = [-,,]';
 obj_old = 1e10;
 tor = 1e-;

 tic

 %%Gradient Descent
 for time = :
     delta = zeros(,);
     objective = ;

     for i = :
         z = x(i,:) * theta;
         h = g(z);%转换成logistic函数
         delta = (/m) .* x(i,:)' * (y(i)-h) + delta;
         objective = (/m) .*( -y(i) * log(h) - (-y(i)) * log(-h)) + objective;
     end
     theta = theta + alpha * delta;

     fprintf('objective is %.4f\n', objective);
     if abs(obj_old - objective) < tor
         fprintf('torlerance is samller than %.4f\n', tor);
         break;
     end
     obj_old = objective;
 end

 %%Calculate the decision boundary line
 plot_x = [min(x(:,)),  max(x(:,))];
 plot_y = (-./theta()).*(theta().*plot_x +theta());
 plot(plot_x, plot_y)
 legend('Admitted', 'Not admitted', 'Decision Boundary')
 hold off
 toc
 pause();
 %%SGD

 figure('NumberTitle', 'off', 'Name', 'SGD');
 plot(x(pos, ), x(pos,), '+');
 hold on;
 plot(x(neg, ), x(neg, ), 'o');

 alpha = 0.001;
 theta = [-,,]';
 obj_old = 1e10;
 tor = 1e-;
 k=;
 U=ceil(m/k);

 for time = :
     delta = zeros(,);
     rand('twister',time*);
     idx=randperm(m);
     objective = ;

     subidx=idx(:k);
     for i=:length(subidx)
         z = x(subidx(i),:) * theta;
         h = g(z);%转换成logistic函数
         delta = (/k) .* x(subidx(i),:)' * (y(subidx(i))-h) + delta;
         objective = (/k) .*( -y(subidx(i)) * log(h) - (-y(subidx(i))) * log(-h)) + objective;
     end
     theta = theta + alpha * delta;

     fprintf('objective is %.4f\n', objective);
     if abs(obj_old - objective) < tor
         fprintf('torlerance is samller than %.4f\n', tor);
         break;
     end
     obj_old = objective;
 end

 %%Calculate the decision boundary line
 plot_x = [min(x(:,)),  max(x(:,))];
 plot_y = (-./theta()).*(theta().*plot_x +theta());
 plot(plot_x, plot_y)
 legend('Admitted', 'Not admitted', 'Decision Boundary')
 hold off
 toc
 pause()

 %%Newton's method

 figure('NumberTitle', 'off', 'Name', 'Newton');
 plot(x(pos, ), x(pos,), '+');
 hold on;
 plot(x(neg, ), x(neg, ), 'o');

 alpha = 0.001;
 theta = zeros(, );
 obj_old = 1e10;
 tor = 1e-;

 for i = :
     delta = zeros(,);
     delta_H = zeros(,);
     objective = ;
     % Calculate the hypothesis function
     for i = :
         z = x(i,:) * theta;
         h = g(z);%转换成logistic函数
         delta = (/m) .* x(i,:)' * (h-y(i)) + delta;
         delta_H = (/m).* x(i,:)' * h * (1-h) * x(i,:) + delta_H;
         objective = (/m) .*( -y(i) * log(h) - (-y(i)) * log(-h)) + objective;
     end
     theta = theta - delta_H\delta;
     fprintf('objective is %.4f\n', objective);
     if abs(obj_old - objective) < tor
         fprintf('torlerance is samller than %.4f\n', tor);
         break;
     end
     obj_old = objective;
 end

 %%Calculate the decision boundary line
 plot_x = [min(x(:,)),  max(x(:,))];
 plot_y = (-./theta()).*(theta().*plot_x +theta());
 plot(plot_x, plot_y)
 legend('Admitted', 'Not admitted', 'Decision Boundary')
 hold off
 toc

 %% Softmax Regression
 close all
 clear

 %%load data
 load('my_ex4x.mat');
 load('my_ex4y.mat');

 [m, n] = size(x);

 % Add intercept term to x
 x = [ones(m, ), x];
 y = y + ;

 class_num = max(y);
 n = n + ;

 %%draw picture
 % find returns the indices of the
 % rows meeting the specified condition
 class2 = find(y == );
 class1 = find(y == );
 class3 = find(y == );
 % Assume the features are in the 2nd and 3rd
 % columns of x
 figure('NumberTitle', 'off', 'Name', 'GD');
 plot(x(class2, ), x(class2,), '+');
 hold on;
 plot(x(class1, ), x(class1, ), 'o');
 hold on;
 plot(x(class3, ), x(class3, ), '*');
 hold on;

 % Define the sigmoid function
 g = inline('exp(z) ./ sumz','z','sumz');

 alpha = 0.0001;
 theta = [-,0.15,0.14;-,,-]';
 obj_old = 1e10;
 tor = 1e-;

 %%Gradient Descent
 for time = :
     delta = zeros(,);
     objective = ;

     for i = :
         for j = :
             z = x(i,:) * theta(:,j);
             sumz = exp(x(i,:) * theta(:,)) +  exp(x(i,:) * theta(:,)) + ;
             h = g(z,sumz);%转换成logistic函数
             if y(i)==j
                 delta = (/m) .* x(i,:)' * (1-h);
                 theta(:,j) = theta(:,j) + alpha * delta;
                 objective = (/m) .*(-y(i) * log(h)) + objective;
             else
                 delta = (/m) .* x(i,:)' * (-h);
                 theta(:,j) = theta(:,j) + alpha * delta;
                 objective = (/m) .*(-(-y(i)) * log(-h)) + objective;
             end
         end
     end

     fprintf('objective is %.4f\n', objective);
     if abs(obj_old - objective) < tor
         fprintf('torlerance is samller than %.4f\n', tor);
         break;
     end
     obj_old = objective;
 end

 %%Calculate the decision boundary line
 plot_x = [min(x(:,)),  max(x(:,))];
 plot_y = (-./theta(,)).*(theta(,).*plot_x +theta(,));
 plot(plot_x, plot_y)
 legend('Admitted', 'Not admitted', 'Decision Boundary')
 hold on

 plot_y = (-./theta(,)).*(theta(,).*plot_x +theta(,));
 plot(plot_x, plot_y)
 legend('Admitted', 'Not admitted', 'Decision Boundary')
 hold off