Exploration of Gradient descent in Lec3 (Jan 20) and Lec4 (Jan 22)

import numpy as np
import matplotlib.pyplot as plt

# Define the function f(x) = x^2 * sin(x) + x
# and f'(x) = 2x * sin(x) + x^2 * cos(x) + 1
def f(x):
    return x**2 * np.sin(x) + x

def fp(x):
    return 2*x*np.sin(x) + x**2 * np.cos(x) + 1

a = 0.05 # learnng rate \alpha 

# x = 2 # (takes many iterations to converge)
x = 5  # converges faster

x = x - a*fp(x)
print(x, fp(x))

5.052840710444088 -0.00045744022560967323

Separable bivariate function $G(x,y) = f(x) + f(y)$

def gradG(vector):
    x, y = vector
    return np.array([fp(x), fp(y)])

vector = np.array([5.0, 5.0])

grd = gradG(vector)
vector = vector - a*grd
print(vector, grd)

[5.05289194 5.05289194] [-0.0022947 -0.0022947]

Nonseparable biavariate function $G(x,y) = x^2 \sin(y)$

def G(x,y):
    return x**2 * np.sin(y)

def GradG(x,y):
    return np.array([2*x*np.sin(y), x**2*np.cos(y)])

vector = np.array([0.05, 1])

grd = GradG(vector[0],vector[1])
vector = vector - a*grd
print(vector, grd)

[0.02475206 0.99968364] [0.04547414 0.00039481]