梯度下降、随机梯度下降、方差减小的梯度下降(matlab实现)

梯度下降代码：

function [ theta, J_history ] = GradinentDecent( X, y, theta, alpha, num_iter )

m = length(y);

J_history = zeros(20, 1);

i = 0;

temp = 0;

for iter = 1:num_iter
     temp = temp +1;
     theta = theta - alpha / m * X' * (X*theta - y);
     if temp>=100
         temp = 0;
         i = i + 1;
         J_history(i) = ComputeCost(X, y, theta);
     end

end

end

随机梯度下降代码：

function [ theta,J_history ] = StochasticGD( X, y, theta, alpha, num_iter )

m = length(y);

J_history = zeros(20, 1);

temp = 0;

n = 0;

for iter = 1:num_iter
     temp = temp + 1;
     index = randi(m);
     theta = theta -alpha *  (X(index, :) * theta - y(index)) * X(index, :)';
     if temp>=100
         temp = 0;
         n = n + 1;
         J_history(n) = ComputeCost(X, y, theta);
     end

end

end

方差减小的梯度下降(SVRG)：

function [ theta_old, J_history ] = SVRG( X, y, theta, alpha )

theta_old = theta;

n = length(y);

J_history = zeros(20,1);

m = 2 * n;

for i = 1:20
     theta_ = theta_old;
     Mu = 1/n *  X' * (X*theta_ - y);
     theta_0 = theta_;
     for j = 1:m
         index = randi(n);
         GD_one = (X(index, :) * theta_0 - y(index)) * X(index, :)';
         GD_ = (X(index, :) * theta_ - y(index)) * X(index, :)';
         theta_t = theta_0 - alpha * (GD_one - GD_ + Mu);
         theta_0 = theta_t;
     end
     J_history(i) = ComputeCost(X, y, theta_t);
     theta_old = theta_t;

end

end

损失函数：

function J = ComputeCost( X, y, theta )

m = length(y);

J = sum((X*theta - y).^2) / (2*m);

end

主程序代码：

%% clean workspace

clc;

clear;

close all;

%% plot data

fprintf('plot data... \n');

X = load('ex2x.dat');

y = load('ex2y.dat');

m = length(y);

figure;

plot(X,y,'o');

%% gradient decent

fprintf('Runing gradient decent... \n');

X = [ones(m,1),X];

theta_SGD = zeros(2, 1);

theta_GD = zeros(2, 1);

theta_SVRG = zeros(2, 1);

Iteration = 2000;

alpha = 0.015;

alpha1 = 0.025;

[theta ,J]= StochasticGD(X, y, theta_SGD, alpha, Iteration);

[theta1 ,J1]= GradinentDecent(X, y, theta_GD, alpha, Iteration);

[theta2 ,J2]= SVRG(X, y, theta_SVRG, alpha1);

fprintf('SGD: %f %f\n',theta(1),theta(2));

fprintf('GD: %f %f\n',theta1(1),theta1(2));

fprintf('SVRG: %f %f\n',theta2(1),theta2(2));

hold on;

plot(X(:, 2), X*theta, 'r-');

plot(X(:, 2), X*theta1, 'g-');

plot(X(:, 2), X*theta2, 'b-');

legend('','SGD','GD','SVRG');

x_j = 1:1:20;

figure;

hold on;

plot(x_j, J, 'b-');

plot(x_j, J1, 'g-');

plot(x_j, J2, 'r-');

legend('SGD','GD','SVRG');

xlabel('epoch')

ylabel('loss')

实验结果：