MSE = (O out e,n Y e,n ) 2 MSE = [ + (o - - PowerPoint PPT Presentation

▶
mse o out e n y e n 2 mse o out 1 1 y 1 1 2 o out 2
SMART_READER_LITE
LIVE PREVIEW

MSE = (O out e,n Y e,n ) 2 MSE = [ + (o - - PowerPoint PPT Presentation

Dec 28, 2022 43 likes •785 views

Feedforward Backpropagation Y [[ 1 0 0] MSE = 0.138 [[ 0 1 0] [[ 0 0 1]] O in O out O delta O error H out T [[ 0.62 0.38 0.64] [[ 0.49 -0.49 0.57] [[-0.38 0.38 0.64] [[-0.09 0.09 0.15] [[ 0.81 0.97 0.97] [[ 0.43 -0.45

slide-1
SLIDE 1

1

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror W2-update = ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ N 1 Odelta) (Hout

T

[[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

slide-2
SLIDE 2

2

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror W2-update = ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ N 1 Odelta) (Hout

T

[[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

slide-3
SLIDE 3

3

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

slide-4
SLIDE 4

4

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

slide-5
SLIDE 5

5

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

slide-6
SLIDE 6

6

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

slide-7
SLIDE 7

7

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

slide-8
SLIDE 8

8

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Oout = sigmoid(Oin) Oin = HoutW2 Hout = sigmoid(Hin) Hin = XW1

slide-9
SLIDE 9

9

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙

slide-10
SLIDE 10

10

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙

slide-11
SLIDE 11

11

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙

slide-12
SLIDE 12

12

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙

slide-13
SLIDE 13

13

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙

slide-14
SLIDE 14

14

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙

Oout = sigmoid(Oin) Hout = sigmoid(Hin) Oin = HoutW2 Hin = XW1

slide-15
SLIDE 15

15

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT

Oout = sigmoid(Oin) Hout = sigmoid(Hin) Oin = HoutW2 Hin = XW1

slide-16
SLIDE 16

16

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT

Oout = sigmoid(Oin) Hout = sigmoid(Hin) Oin = HoutW2 Hin = XW1

slide-17
SLIDE 17

17

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT

Oout = sigmoid(Oin) Hout = sigmoid(Hin) Oin = HoutW2 Hin = XW1

slide-18
SLIDE 18

18

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙

Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] Oerror = Oout – Y Oout (1 – Oout) ⊙ [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂W1 ∂MSE(W1) :

∂ MSE ∂ W2 ∂ MSE ∂ W1 (W1) = ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ W2 ∂ MSE ∂ Oout ∂ Oout ∂ Oin ∂ Oin ∂ Hout ∂ Hout ∂ Hin ∂ Hin ∂ W1 (W2) =

W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2 ?

Oout = sigmoid(Oin) Hout = sigmoid(Hin) Oin = HoutW2 Hin = XW1

slide-19
SLIDE 19

19

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N

x1 x3 x2 MSE x4 W1 W2 Hout Hin Oout Oin X

Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1

slide-20
SLIDE 20

20

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N

x1 x3 x2 MSE x4 W1 W2 Hout Hin Oout Oin X

Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 Hin = XW1 Hout = sigmoid(Hin)

slide-21
SLIDE 21

21

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 Hin = XW1 Hout = sigmoid(Hin)

slide-22
SLIDE 22

22

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 Hin = XW1 Hout = s(Hin)

x1 x3 x2 MSE x4 W1 W2 Hout Hin Oout Oin X

hin 1

(1) =

slide-23
SLIDE 23

23

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 Hin = XW1 Hout = s(Hin)

x1 x3 x2 MSE x4 W1 W2 Hout Hin Oout Oin X

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

slide-24
SLIDE 24

24

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 Hin = XW1 Hout = s(Hin)

x1 x3 x2 MSE x4 W1 W2 Hout Hin Oout Oin X

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

slide-25
SLIDE 25

25

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 Hin = XW1 Hout = s(Hin)

x1 x3 x2 MSE x4 W1 W2 Hout Hin Oout Oin X

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

hin 1

(2) = x1 (2) w1-11 + x2 (2) w1-21 + x3 (2) w1-31 + x4 (2) w1-41

hin 2

(2) = x1 (2) w1-12 + x2 (2) w1-22 + x3 (2) w1-32 + x4 (2) w1-42

slide-26
SLIDE 26

26

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 Hin = XW1 Hout = s(Hin)

x1 x3 x2 MSE x4 W1 W2 Hout Hin Oout Oin X

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

hin 1

(2) = x1 (2) w1-11 + x2 (2) w1-21 + x3 (2) w1-31 + x4 (2) w1-41

hin 2

(2) = x1 (2) w1-12 + x2 (2) w1-22 + x3 (2) w1-32 + x4 (2) w1-42

hout 1

(1) = sigmoid(hin 1 (1))

hout 2

(1) = sigmoid(hin 2 (1))

hout 1

(2) = sigmoid(hin 1 (2))

hout 2

(2) = sigmoid(hin 2 (2))

slide-27
SLIDE 27

27

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 hout 1

(1) = sigmoid(hin 1 (1))

hout 2

(1) = sigmoid(hin 2 (1))

hout 1

(2) = sigmoid(hin 1 (2))

hout 2

(2) = sigmoid(hin 2 (2))

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

Hin = XW1 Hout = s(Hin) hin 1

(2) = x1 (2) w1-11 + x2 (2) w1-21 + x3 (2) w1-31 + x4 (2) w1-41

hin 2

(2) = x1 (2) w1-12 + x2 (2) w1-22 + x3 (2) w1-32 + x4 (2) w1-42

∂MSE ∂w1-11 =

slide-28
SLIDE 28

28

Scalar-notation

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • in 1

(2) = hout 1 (2) w2-11 + hout 2 (2) w2-21

  • in 2

(2) = hout 1 (2) w2-12 + hout 2 (2) w2-22

  • in 3

(2) = hout 1 (2) w2-13 + hout 2 (2) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

  • out 1

(2) = sigmoid(oin 1 (2))

  • out 2

(2) = sigmoid(oin 2 (2))

  • out 3

(2) = sigmoid(oin 3 (2))

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N Oin = HoutW2 Oout = sigmoid(Oin)

Matrix-notation

ΣΣ

e n

MSE = (Oout e,n – Ye,n)2 2N 1 hout 1

(1) = sigmoid(hin 1 (1))

hout 2

(1) = sigmoid(hin 2 (1))

hout 1

(2) = sigmoid(hin 1 (2))

hout 2

(2) = sigmoid(hin 2 (2))

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

Hin = XW1 Hout = s(Hin) hin 1

(2) = x1 (2) w1-11 + x2 (2) w1-21 + x3 (2) w1-31 + x4 (2) w1-41

hin 2

(2) = x1 (2) w1-12 + x2 (2) w1-22 + x3 (2) w1-32 + x4 (2) w1-42

Multivariable Chain Rule

z(x, y) x(u, v) y(u, v) ∂z ∂u = ∂z ∂x ∂x ∂u ∂z ∂y ∂y ∂u + z x y u v u v ∂z ∂v = ∂z ∂x ∂x ∂v ∂z ∂y ∂y ∂v +

∂MSE ∂w1-11 =

slide-29
SLIDE 29

29

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 =

slide-30
SLIDE 30

30

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 =

slide-31
SLIDE 31

31

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11

slide-32
SLIDE 32

32

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11

slide-33
SLIDE 33

33

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ...

slide-34
SLIDE 34

34

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂w1-11 ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 = + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂MSE

slide-35
SLIDE 35

35

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂w1-11 ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 = + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂MSE 1 N = (oout 1

(1) – y1 (1)) oout 1 (1) (1 - oout 1 (1))

slide-36
SLIDE 36

36

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂w1-11 ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 = + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂MSE 1 N = (oout 1

(1) – y1 (1)) oout 1 (1) (1 - oout 1 (1))

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

hout 1

(1) = sigmoid(hin 1 (1))

hout 2

(1) = sigmoid(hin 2 (1))

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N

slide-37
SLIDE 37

37

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂w1-11 ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 = + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂MSE 1 N

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

hout 1

(1) = sigmoid(hin 1 (1))

hout 2

(1) = sigmoid(hin 2 (1))

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N = (oout 1

(1) – y1 (1)) oout 1 (1) (1 - oout 1 (1)) w2-11

slide-38
SLIDE 38

38

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂w1-11 ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 = + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂MSE

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

hout 1

(1) = sigmoid(hin 1 (1))

hout 2

(1) = sigmoid(hin 2 (1))

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N 1 N = (oout 1

(1) – y1 (1)) oout 1 (1) (1 - oout 1 (1)) w2-11 hout 1 (1) (1 - hout 1 (1))

slide-39
SLIDE 39

39

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂w1-11 ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 = + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂MSE

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

hout 1

(1) = sigmoid(hin 1 (1))

hout 2

(1) = sigmoid(hin 2 (1))

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N 1 N = (oout 1

(1) – y1 (1)) oout 1 (1) (1 - oout 1 (1)) w2-11 hout 1 (1) (1 - hout 1 (1)) x1 (1)

slide-40
SLIDE 40

40

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂w1-11 ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 = + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂MSE

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • out 1

(1) = sigmoid(oin 1 (1))

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

  • out 2

(1) = sigmoid(oin 2 (1))

  • out 3

(1) = sigmoid(oin 3 (1))

hout 1

(1) = sigmoid(hin 1 (1))

hout 2

(1) = sigmoid(hin 2 (1))

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

MSE = [ + (oout 1

(1) – y1 (1))2 + (oout 2 (1) – y2 (1))2 + (oout 3 (1) – y3 (1))2

MSE = + (oout 1

(2) – y1 (2))2 + (oout 2 (2) – y2 (2))2 + (oout 3 (2) – y3 (2))2 ]

1 2N 1 N = (oout 1

(1) – y1 (1)) oout 1 (1) (1 - oout 1 (1)) w2-11 hout 1 (1) (1 - hout 1 (1)) x1 (1)

1 N = (oout 2

(1) – y2 (1)) oout 2 (1) (1 - oout 2 (1)) w2-12 hout 1 (1) (1 - hout 1 (1)) x1 (1)

1 N = (oout 3

(1) – y3 (1)) oout 3 (1) (1 - oout 3 (1)) w2-13 hout 1 (1) (1 - hout 1 (1)) x1 (1)

+ +

slide-41
SLIDE 41

41

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... 1 N = (oout 1

(1) – y1 (1)) oout 1 (1) (1 - oout 1 (1)) w2-11 hout 1 (1) (1 - hout 1 (1)) x1 (1)

1 N = (oout 2

(1) – y2 (1)) oout 2 (1) (1 - oout 2 (1)) w2-12 hout 1 (1) (1 - hout 1 (1)) x1 (1)

1 N = (oout 3

(1) – y3 (1)) oout 3 (1) (1 - oout 3 (1)) w2-13 hout 1 (1) (1 - hout 1 (1)) x1 (1)

+ +

slide-42
SLIDE 42

42

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... 1 N + +

Σ [

=

]

= (oout 1

(1) – y1 (1)) oout 1 (1) (1 - oout 1 (1)) w2-11 hout 1 (1) (1 - hout 1 (1)) x1 (1)

= (oout 2

(1) – y2 (1)) oout 2 (1) (1 - oout 2 (1)) w2-12 hout 1 (1) (1 - hout 1 (1)) x1 (1)

= (oout 3

(1) – y3 (1)) oout 3 (1) (1 - oout 3 (1)) w2-13 hout 1 (1) (1 - hout 1 (1)) x1 (1)

slide-43
SLIDE 43

43

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... 1 N = (oout 1

(e) – y1 (e)) oout 1 (e) (1 - oout 1 (e)) w2-11 hout 1 (e) (1 - hout 1 (e)) x1 (e)

= (oout 2

(e) – y2 (e)) oout 2 (e) (1 - oout 2 (e)) w2-12 hout 1 (e) (1 - hout 1 (e)) x1 (e)

= (oout 3

(e) – y3 (e)) oout 3 (e) (1 - oout 3 (e)) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

=

]

slide-44
SLIDE 44

44

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... 1 N = (oout 1

(e) – y1 (e)) oout 1 (e) (1 - oout 1 (e)) w2-11 hout 1 (e) (1 - hout 1 (e)) x1 (e)

= (oout 2

(e) – y2 (e)) oout 2 (e) (1 - oout 2 (e)) w2-12 hout 1 (e) (1 - hout 1 (e)) x1 (e)

= (oout 3

(e) – y3 (e)) oout 3 (e) (1 - oout 3 (e)) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

=

]

∂W2 ∂MSE(W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2 ?

slide-45
SLIDE 45

45

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂W2 ∂MSE(W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2 ? 1 N = (oout 1

(e) – y1 (e)) oout 1 (e) (1 - oout 1 (e)) w2-11 hout 1 (e) (1 - hout 1 (e)) x1 (e)

= (oout 2

(e) – y2 (e)) oout 2 (e) (1 - oout 2 (e)) w2-12 hout 1 (e) (1 - hout 1 (e)) x1 (e)

= (oout 3

(e) – y3 (e)) oout 3 (e) (1 - oout 3 (e)) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

=

]

slide-46
SLIDE 46

46

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂W2 ∂MSE(W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2 ? 1 N = (oout 1

(e) – y1 (e)) oout 1 (e) (1 - oout 1 (e)) w2-11

= (oout 2

(e) – y2 (e)) oout 2 (e) (1 - oout 2 (e)) w2-12

= (oout 3

(e) – y3 (e)) oout 3 (e) (1 - oout 3 (e)) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

=

]

slide-47
SLIDE 47

47

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂W2 ∂MSE(W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2 ? 1 N = (oout 1

(e) – y1 (e)) oout 1 (e) (1 - oout 1 (e)) w2-11

= (oout 2

(e) – y2 (e)) oout 2 (e) (1 - oout 2 (e)) w2-12

= (oout 3

(e) – y3 (e)) oout 3 (e) (1 - oout 3 (e)) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

=

]

slide-48
SLIDE 48

48

MSE hout 1

(1)

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

w1-11 w1-21

  • out 1

(2)

w1-31

  • out 2

(2)

  • out 3

(2)

hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

  • out 2

(1)

∂MSE ∂w1-11 = ∂MSE ∂oout 1

(1)

doout 1

(1)

doin 1

(1)

∂oin 1

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 2

(1)

doout 2

(1)

doin 2

(1)

∂oin 2

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ∂MSE ∂oout 3

(1)

doout 3

(1)

doin 3

(1)

∂oin 3

(1)

∂hout 1

(1) dhin 1 (1)

dhout 1

(1)

∂hin 1

(1)

∂w1-11 + ... ∂W2 ∂MSE(W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : W2-update = N 1 Odelta) (Hout

T

Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2 ? 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

=

]

slide-49
SLIDE 49

49

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

hout 1

(1)

w1-11 w1-21 w1-31 hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

MSE

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

  • out 1

(2) oout 2 (2)

  • out 3

(2)

  • out 2

(1)

∂MSE ∂w1-12 =

slide-50
SLIDE 50

50

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

hout 1

(1)

w1-11 w1-21 w1-31 hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

MSE

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

  • out 1

(2) oout 2 (2)

  • out 3

(2)

  • out 2

(1)

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

slide-51
SLIDE 51

51

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

hout 1

(1)

w1-11 w1-21 w1-31 hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

MSE

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

  • out 1

(2) oout 2 (2)

  • out 3

(2)

  • out 2

(1)

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

  • in 1

(1) = hout 1 (1) w2-11 + hout 2 (1) w2-21

  • in 2

(1) = hout 1 (1) w2-12 + hout 2 (1) w2-22

  • in 3

(1) = hout 1 (1) w2-13 + hout 2 (1) w2-23

slide-52
SLIDE 52

52

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

hout 1

(1)

w1-11 w1-21 w1-31 hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

MSE

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

  • out 1

(2) oout 2 (2)

  • out 3

(2)

  • out 2

(1)

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

hin 1

(1) = x1 (1) w1-11 + x2 (1) w1-21 + x3 (1) w1-31 + x4 (1) w1-41

hin 2

(1) = x1 (1) w1-12 + x2 (1) w1-22 + x3 (1) w1-32 + x4 (1) w1-42

slide-53
SLIDE 53

53

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

hout 1

(1)

w1-11 w1-21 w1-31 hout 2

(1)

hin 1

(1)

hin 2

(1)

w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

hout 1

(1)

hout 2

(1)

hin 1

(1)

hin 2

(1)

MSE

  • out 1

(1)

  • in 1

(1)

  • in 2

(1)

  • out 3

(1)

  • in 3

(1)

  • out 1

(2) oout 2 (2)

  • out 3

(2)

  • out 2

(1)

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂W2 (W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2 ? W2-update = N 1 Odelta) (Hout

T

slide-54
SLIDE 54

54

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

54 54

slide-55
SLIDE 55

55

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

55 55

slide-56
SLIDE 56

56

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

56 56

slide-57
SLIDE 57

57

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

57 57

slide-58
SLIDE 58

58

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

58 58

slide-59
SLIDE 59

59

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

59 59

slide-60
SLIDE 60

60

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

] Herror

[[ herror 1

(1) herror 2 (1)]

[[ herror 1

(2) herror 2 (2)]

[[ herror 1

(3) herror 2 (3)]]

60 60

slide-61
SLIDE 61

61

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

] Herror

[[ herror 1

(1) herror 2 (1)]

[[ herror 1

(2) herror 2 (2)]

[[ herror 1

(3) herror 2 (3)]]

61 61

slide-62
SLIDE 62

62

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

∂MSE ∂w1-11 = 1 N = odelta 1

(e) w2-11

= odelta 2

(e) w2-12

= odelta 3

(e) w2-13 hout 1 (e) (1 - hout 1 (e)) x1 (e)

+ +

Σ [

e

]

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

] Herror

[[ herror 1

(1) herror 2 (1)]

[[ herror 1

(2) herror 2 (2)]

[[ herror 1

(3) herror 2 (3)]]

62 62

slide-63
SLIDE 63

63

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update Herror

[[ herror 1

(1) herror 2 (1)]

[[ herror 1

(2) herror 2 (2)]

[[ herror 1

(3) herror 2 (3)]]

∂MSE ∂w1-11 = 1 N herror 1

(e) hout 1 (e) (1 - hout 1 (e)) x1 (e)

Σ

e

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

63 63

slide-64
SLIDE 64

64

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update Herror

[[ herror 1

(1) herror 2 (1)]

[[ herror 1

(2) herror 2 (2)]

[[ herror 1

(3) herror 2 (3)]]

∂MSE ∂w1-11 = 1 N herror 1

(e) hout 1 (e) (1 - hout 1 (e)) x1 (e)

Σ

e

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

64 64

slide-65
SLIDE 65

65

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update Herror

[[ herror 1

(1) herror 2 (1)]

[[ herror 1

(2) herror 2 (2)]

[[ herror 1

(3) herror 2 (3)]]

∂MSE ∂w1-11 = 1 N herror 1

(e) hout 1 (e) (1 - hout 1 (e)) x1 (e)

Σ

e

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

65 65

slide-66
SLIDE 66

66

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update Herror

[[ herror 1

(1) herror 2 (1)]

[[ herror 1

(2) herror 2 (2)]

[[ herror 1

(3) herror 2 (3)]]

∂MSE ∂w1-11 = 1 N herror 1

(e) hout 1 (e) (1 - hout 1 (e)) x1 (e)

Σ

e

∂MSE ∂w1-12 = 1 N = odelta 1

(e) w2-21

= odelta 2

(e) w2-22

= odelta 3

(e) w2-23 hout 2 (e) (1 - hout 2 (e)) x1 (e)

+ +

Σ [

e

]

66 66

slide-67
SLIDE 67

67

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update Herror

[[ herror 1

(1) herror 2 (1)]

[[ herror 1

(2) herror 2 (2)]

[[ herror 1

(3) herror 2 (3)]]

∂MSE ∂w1-11 = 1 N herror 1

(e) hout 1 (e) (1 - hout 1 (e)) x1 (e)

Σ

e

∂MSE ∂w1-12 = 1 NΣ

e herror 2

(e) hout 2 (e) (1 - hout 2 (e)) x1 (e)

67 67

slide-68
SLIDE 68

68

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

[[ 0.0 0.6] [[-0.1 -0.5] [[ 0.2 0.5]]

W2

T

w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hin 1

(1)

hin 2

(1)

∂MSE ∂W2 (W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2 ? W2-update = N 1 Odelta) (Hout

T

68 68

slide-69
SLIDE 69

69

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

[[ 0.0 0.6] [[-0.1 -0.5] [[ 0.2 0.5]]

W2

T

w1-11 w1-21 w1-31 w1-41 w1-12 w1-22 w1-32 w1-42 hin 1

(1)

hin 2

(1)

∂MSE ∂W2 (W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2

T

W2-update = N 1 Odelta) (Hout

T

69 69

slide-70
SLIDE 70

70

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

[[ 0.0 0.6] [[-0.1 -0.5] [[ 0.2 0.5]]

W2

T

∂MSE ∂W2 (W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2

T

W2-update = N 1 Odelta) (Hout

T

70 70

slide-71
SLIDE 71

71

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

[[ 0.0 0.6] [[-0.1 -0.5] [[ 0.2 0.5]]

W2

T

∂MSE ∂W2 (W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2

T

W2-update = N 1 Odelta) (Hout

T

Herror

[[ 0.021 -0.024] [[ 0.045 0.240] [[-0.028 -0.011]]

71 71

slide-72
SLIDE 72

72

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

[[ 0.0 0.6] [[-0.1 -0.5] [[ 0.2 0.5]]

W2

T

∂MSE ∂W2 (W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2

T

W2-update = N 1 Odelta) (Hout

T

Herror

[[ 0.021 -0.024] [[ 0.045 0.240] [[-0.028 -0.011]]

Hdelta

[[ 0.003 -0.004] [[ 0.001 0.049] [[-0.001 -0.003]]

72 72

slide-73
SLIDE 73

73

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

[[ 0.0 0.6] [[-0.1 -0.5] [[ 0.2 0.5]]

W2

T

∂MSE ∂W2 (W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2

T

W2-update = N 1 Odelta) (Hout

T

Herror

[[ 0.021 -0.024] [[ 0.045 0.240] [[-0.028 -0.011]]

Hdelta

[[ 0.003 -0.004] [[ 0.001 0.049] [[-0.001 -0.003]] [[ 4.9 6.4 5.8] [[ 3.0 3.2 2.7] [[ 1.4 4.5 5.1] [[ 0.2 1.5 1.9]]

XT XT

[[ 0.002 0.031] [[ 0.001 0.015] [[ 0.000 0.022] [[ 0.000 0.007]]

W1-update

73 73

slide-74
SLIDE 74

74

x1 x3 x2

[[ 4.9 3.0 1.4 0.2] [ 6.4 3.2 4.5 1.5] [ 5.8 2.7 5.1 1.9]]

W1 W2 X

x4

[[ 0.7 0.4] [[-0.8 0.0] [[ 0.3 -0.4] [[ 0.1 0.1]] [[ 0.0 -0.1 0.2] [[ 0.6 -0.5 0.5]]

Hin Hout

[[ 1.47 1.42] [[ 3.42 0.91] [[ 3.62 0.47]] [[ 0.81 0.81] [[ 0.97 0.71] [[ 0.97 0.62]]

Oin Oout

[[ 0.49 -0.49 0.57] [[ 0.43 -0.45 0.55] [[ 0.37 -0.41 0.50]] [[ 0.62 0.38 0.64] [[ 0.61 0.39 0.63] [[ 0.59 0.40 0.62]]

Feedforward MSE = 0.138 Y

[[ 1 0 0] [[ 0 1 0] [[ 0 0 1]]

Backpropagation Oerror

[[-0.38 0.38 0.64] [[ 0.61 -0.61 0.63] [[ 0.59 0.40 -0.38]]

Odelta

[[-0.09 0.09 0.15] [[ 0.15 -0.15 0.15] [[ 0.14 0.10 -0.09]]

Hout

T

[[ 0.81 0.97 0.97] [[ 0.81 0.71 0.62]] [[0.023 0.003 0.020] [[0.013 0.003 0.019]]

W2-update

[[ 0.0 0.6] [[-0.1 -0.5] [[ 0.2 0.5]]

W2

T

∂MSE ∂W2 (W2) : Odelta = Oerror ⊙ Oerror = Oout – Y Oout (1 – Oout) ⊙ ∂W1 ∂MSE(W1) : Hdelta = Herror ⊙ Hout (1 – Hout) ⊙ W1-update = N 1 Hdelta) (XT Herror = Odelta W2

T

W2-update = N 1 Odelta) (Hout

T

Herror

[[ 0.021 -0.024] [[ 0.045 0.240] [[-0.028 -0.011]]

Hdelta

[[ 0.003 -0.004] [[ 0.001 0.049] [[-0.001 -0.003]] [[ 4.9 6.4 5.8] [[ 3.0 3.2 2.7] [[ 1.4 4.5 5.1] [[ 0.2 1.5 1.9]]

XT XT

[[ 0.002 0.031] [[ 0.001 0.015] [[ 0.000 0.022] [[ 0.000 0.007]]

W1-update

Wi := Wi – α (XWi)

Gradient Descent:

∂ MSE ∂ Wi 74 74