hw4

February 1, 2024

1 Homework 4
Solve exercises 10.1, 10.2, 10.15, 11.15
In the next cell, we copy all the functions from the last homework. We will use them for comparison
with the new ones.
[ ]: function LU(A)
n, m = size(A) # A is supposed to be a square matrix, so hopefully n and m␣
↪will be equal.

# We initalize L with zeros and U to be the same as A.

L = zeros(n,m)
U = copy(A)

for k in 1:n
L[k,k] = 1
for i in (k+1):n
L[i,k] = U[i,k]/U[k,k]
U[i,:] = U[i,:] - L[i,k]*U[k,:]
end
end
return L, U
end

function backward_substitute(U,y)
n, m = size(U)
r, = size(y)
@assert n == m == r
x = zeros(n)

for i in n:-1:1
tail = 0
for j in i+1:n
tail = tail + U[i,j]*x[j]
end
x[i] = (y[i] - tail)/U[i,i]
end

1
 return x
end

function forward_substitute(L,y)
n, m = size(L)
r, = size(y)
@assert n == m == r
x = zeros(n)
for i in 1:n
tail = 0
for j in 1:i-1
tail = tail + L[i,j]*x[j]
end
x[i] = (y[i] - tail)/L[i,i]
end
return x
end

function solve(A,f)
n, m = size(A)
r, = size(f)
@assert n == m == r

# Something goes here

L, U = LU(A)
y = forward_substitute(L,f)
x = backward_substitute(U,y)

return(x)
end

function big_matrix(n)
A = zeros(n^2,n^2)
for i in 1:n^2
A[i,i] = 4
if mod(i,n)!=0
A[i+1,i] = -1
A[i,i+1] = -1
end
if i<=n^2-n
A[i,i+n] = -1
A[i+n,i] = -1
end
end
return A
end

2
function big_rhs(n, f)
y = zeros(n^2)
for i in 1:n
y[i] += f(i/(n+1),0.)
y[n^2-n+i] += f(i/(n+1),1.)
end

for i in 1:n
y[n*i] += f(1.,i/(n+1))
y[n*i-n+1] += f(0.,i/(n+1))
end
return y
end

function T(u::Array{<:Real,2})
n, m = size(u)
v = similar(u) # This line creates another array with the same dimensions␣
↪and type as u

for i in 1:n
for j in 1:m
v[i,j] = 4*u[i,j]
if i>1 v[i,j] -= u[i-1,j] end
if i<n v[i,j] -= u[i+1,j] end
if j>1 v[i,j] -= u[i,j-1] end
if j<m v[i,j] -= u[i,j+1] end
end
end
return v
end

function matrix_rhs(n::Integer, f::Function)

b = zeros(n,n)
for i in 1:n
b[1,i] += f(0.,i/(n+1))
b[n,i] += f(1.,i/(n+1))
b[i,1] += f(i/(n+1),0.)
b[i,n] += f(i/(n+1),1.)
end
return b
end

function gauss_seidel_iteration!(u::Array{<:Real,2}, b::Array{<:Real,2})

n, m = size(u)
for i in 1:n
for j in 1:m
s = b[i,j]

3
 if i>1 s += u[i-1,j] end
if j>1 s += u[i,j-1] end
if i<n s += u[i+1,j] end
if j<m s += u[i,j+1] end
u[i,j] = s/4
end
end
end

function gauss_seidel(u::Array{<:Real,2}, b::Array{<:Real,2}, iterations::

↪Integer)

for i in 1:iterations
gauss_seidel_iteration!(u,b)
end
return u
end

function dot_product(u::Array{<:Real,2}, v::Array{<:Real,2})

return sum(u.*v)
end

function gradient_descent(u::Array{<:Real,2}, b::Array{<:Real,2}, iterations::

↪Integer)

for i in 1:iterations
r = T(u) - b
a = dot_product(r,r) / dot_product(T(r),r)
u = u - a*r
end
return u
end

[ ]: gradient_descent (generic function with 1 method)

Now you are supposed to implement the conjugate gradients method.

[ ]: function conjugate_gradients(y::Array{<:Real,2}, g::Array{<:Real,2}, iterations:
↪:Integer)

r = T(y) - g
s = T(y) - g
z = T(s)
for i in 1:iterations
alpha = -(dot_product(r,s) / dot_product(s,T(s)))
y = y + alpha*s

z = T(s)
r = r + alpha*z
if supnorm(r) < 1e-60

4
 break
end
beta = -(dot_product(r,z) / dot_product(s,z)) # Differs from wikipedia

s = r + beta*s

end
return y
end

[ ]: conjugate_gradients (generic function with 1 method)

Let us test how long these algorithms take in practice. So, we will time their execution time with
a 40x40 mesh.
[ ]: n = 40
f(x,y) = 1 - sin(pi*x)
partition = 1/(n+1) : 1/(n+1) : 1-1/(n+1)

# We use the following function to roughly measure the size of vectors or␣
↪matrices.

function supnorm(u)
s = 0
for entry in u
s = max(abs(entry),s)
end
return s
end

# Using Gaussian elimination

y = big_rhs(n,f)
A = big_matrix(n)
@time x = solve(A,y)
println("Accuracy of Gaussian elimination: ", supnorm(A*x-y) / n^2)

# Using Gauss-Seidel iteration

# We can test different number of iterations

iterations = div(n^2,10)
b = matrix_rhs(n,f)
u = zeros(n,n)
@time u = gauss_seidel(u,b,iterations)
println("Accuracy of Gauss Seidel with ", iterations, " iterations: ",␣
↪supnorm(T(u)-b)/n^2)

iterations = n^2
b = matrix_rhs(n,f)

5
u = zeros(n,n)
@time u = gauss_seidel(u,b,iterations)
println("Accuracy of Gauss Seidel with ", iterations, " iterations: ",␣
↪supnorm(T(u)-b)/n^2)

iterations = 10*n^2
b = matrix_rhs(n,f)
u = zeros(n,n)
@time u = gauss_seidel(u,b,iterations)
println("Accuracy of Gauss Seidel with ", iterations, " iterations: ",␣
↪supnorm(T(u)-b)/n^2)

# Using Gradient descent

# We can test different number of iterations

iterations = n^2
b = matrix_rhs(n,f)
u = zeros(n,n)
@time u = gradient_descent(u,b,iterations)
println("Accuracy of Gradient descent with ", iterations, " iterations: ",␣
↪supnorm(T(u)-b)/n^2)

# Using Conjugate radients

# We can test different number of iterations

iterations = div(n^2,10)
b = matrix_rhs(n,f)
u = zeros(n,n)
@time u = conjugate_gradients(u,b,iterations)
println("Accuracy of conjugate gradients with ", iterations, " iterations: ",␣
↪supnorm(T(u)-b)/n^2)

iterations = n^2
b = matrix_rhs(n,f)
u = zeros(n,n)
@time u = conjugate_gradients(u,b,iterations)
println("Accuracy of conjugate gradients with ", iterations, " iterations: ",␣
↪supnorm(T(u)-b)/n^2)

### For some reason I am unable to do conjugate gradients past 1297 iterations␣
↪no matter what.

11.718718 seconds (5.12 M allocations: 61.645 GiB, 18.19% gc time)

Accuracy of Gaussian elimination: 1.6653345369377347e-18
0.000902 seconds
Accuracy of Gauss Seidel with 160 iterations: 2.6186415244190688e-6
0.008966 seconds

6
 Accuracy of Gauss Seidel with 1600 iterations: 5.985804765046288e-10
0.088046 seconds
Accuracy of Gauss Seidel with 16000 iterations: 4.163336342344337e-19
0.024841 seconds (11.20 k allocations: 138.086 MiB, 20.52% gc time)
Accuracy of Gradient descent with 1600 iterations: 9.809589988279965e-8
0.009217 seconds (7.80 k allocations: 24.120 MiB, 55.81% compilation time:
100% of which was recompilation)
Accuracy of conjugate gradients with 160 iterations: 3.0531133177191805e-18
0.014835 seconds (5.80 k allocations: 71.472 MiB, 19.28% gc time)
Accuracy of conjugate gradients with 1600 iterations: 3.0531133177191805e-18
Conjugate gradients wins by a large margin. In this example, we are getting the exact solution
in even fewer iterations than expected. I don’t have a good explanation of why, and we shouldn’t
expect that in general.
We can even solve the problem with a massive mesh of 200x200. It takes less than a minute to
get the exact solution in my (old) office computer. Good luck doing that with any of the other
methods.
[ ]: n = 200
f(x,y) = 1 - sin(pi*x)
partition = 1/(n+1) : 1/(n+1) : 1-1/(n+1)
iterations = n^2
b = matrix_rhs(n,f)
u = zeros(n,n)
@time u = conjugate_gradients(u,b,iterations)
println("Accuracy of conjugate gradients with ", iterations, " iterations: ",␣
↪supnorm(T(u)-b)/n^2)

1.749886 seconds (57.05 k allocations: 8.502 GiB, 11.67% gc time)

Accuracy of conjugate gradients with 40000 iterations: 3.441691376337985e-19

[ ]:

1.1 Fun movie

Let us make a movie to compare the speed of gradient descent with conjugate gradients.
[ ]: using Plots
iterations = 5
n = 40
partition = 1/(n+1) : 1/(n+1) : 1-1/(n+1)

let b = matrix_rhs(n,f), u = zeros(n,n), v = zeros(n,n)

@gif for i in 1:300
u = gradient_descent(u,b,iterations)
v = conjugate_gradients(v,b,iterations)
sgd1 = surface(partition,partition,u, title=string(i*iterations)*"␣
↪iterations")

7
 sgd2 = surface(partition,partition,v, title=string(i*iterations)*"␣
iterations")
↪

sg = plot(sgd1,sgd2,layout=2, size=(800,300))
end
end

[ Info: Saved animation to

/Users/bradleyyu/Documents/Coding-Things/MATH212/tmp.gif

[ ]: Plots.AnimatedGif("/Users/bradleyyu/Documents/Coding-Things/MATH212/tmp.gif")

2 Interpolation polynomials
Let us explore what we get when we interpolate a function with a polynomial. Let us start with a
function that computes Newton’s divided differences.

newton(𝑓, [𝑥1 , … , 𝑥𝑛 ]) ∶= 𝑓[𝑥1 , … , 𝑥𝑛 ]

In order to implement the function below, we use the following formula for 𝑛 > 1,
𝑓[𝑥1 , … , 𝑥𝑛−1 ] − 𝑓[𝑥1 , … , 𝑥𝑛−2 , 𝑥𝑛 ]
𝑓[𝑥1 , … , 𝑥𝑛 ] = .
𝑥𝑛−1 − 𝑥𝑛

With this formula, it is relatively easy to implement newton recursively. I include the code below.
[ ]: function newton(f::Function, x::Vector{<:Real})
n = length(x)
if (n==1) return f(x[1]) end
yn = x[1:end-1]
yl = vcat(x[1:end-2],[x[n]])
return (newton(f,yn)-newton(f,yl))/(x[n-1]-x[n])
end

[ ]: newton (generic function with 1 method)

We load a Julia package called Polynomials. It is used to make computations with polynomials as
the name suggests.
[ ]: using Polynomials

The polynomials can be defined from a list of coefficients. Arguably, it looks better to define a
global variable x that refers to the 𝑥 polynomial and then do basic arithmetic with it. Let us see.
[ ]: global x = Polynomial([0,1])
display(1 + 2x + 3x^2 + 5x^3)
display((x-1.5)*(x-2))

1 + 2 ⋅ 𝑥 + 3 ⋅ 𝑥2 + 5 ⋅ 𝑥3

8
 3.0 − 3.5 ⋅ 𝑥 + 1.0 ⋅ 𝑥2

2.1 Exercise
Implement a function that returns a polynomial that interpolates f on any given collection of real
numbers {x1, x2, …, xn}.

[ ]: function interpolate(f::Function, points::Vector{<:Real})

n = length(points) # note that the polynomial will be of degree n-1
p = Polynomial(0)
# Lagrange Interpolation
for i in 1:n
l = Polynomial(1)
for j in 1:n
if i != j
l = l * (x - points[j]) / (points[i] - points[j])
end
end
p = p + f(points[i]) * l
end

return p
end

[ ]: interpolate (generic function with 1 method)

[ ]: # Let us test if it seems to be working well

f(t) = sin(2*�*t)
sample = Array(0:0.25:1)
p = interpolate(f,sample)
xg = 0:0.01:1
plot(xg,map(p,xg), label="Lagrange interpolation")
plot!(f,0,1, label="sin(2 pi x)")
scatter!(sample, map(f,sample), label="Interpolation points")
[ ]:

9
[ ]: # Let us now try to interpolate the Gaussian function
# The result should be mostly fine, with some artifacts near the end of the␣
↪interval.

f(t) = exp(-t^2)
sample = Array(-3:0.5:3)
p = interpolate(f,sample)
xg = -3:0.01:3
plot(xg,map(p,xg), label="Lagrange interpolation")
plot!(f,-3,3, label="exp(-x^2)")
scatter!(sample, map(f,sample), label="Interpolation points")
[ ]:

10
[ ]: # Let us now try to interpolate the Runge function. It is an example whose␣
↪higher derivatives grow.

# The result should be pretty bad near the end points of the interval.

f(t) = 1/(1+t^2)
sample = Array(-3:0.5:3)
p = interpolate(f,sample)
xg = -3:0.01:3
plot(xg,map(p,xg), label="Lagrange interpolation")
plot!(f,-3,3, label="1/(1+x^2)")
scatter!(sample, map(f,sample), label="Interpolation points")
[ ]:

11
12