using Distributions
using DelimitedFiles
using LinearAlgebra
using CuArrays
use_gpu = true # helper to easily switch between gpu/cpu
todevice(x) = use_gpu ? gpu(x) : x
todevice (generic function with 1 method)
# Activation function
sigmoid(x) = 1 / (1 + exp(-x))
sigmoid (generic function with 1 method)
mutable struct NetInfo
# Input Layer, Hidden Layer and Output Layer nodes numbers
inodes::Int
hnodes::Int
onodes::Int
# Link weight matrix
wih::Array{Float64, 2} # from input layer to hidden layer
who::Array{Float64, 2} # from hidden layer to output layer
# learning rate
lr::Float64
end
# Factory method - Init Net
function InitNet(inodes::Int, hnodes::Int, onodes::Int, lr::Float64)
wih = rand(Normal(0, hnodes^(-0.5)), hnodes, inodes)
who = rand(Normal(0, onodes^(-0.5)), onodes, hnodes)
return NetInfo(inodes, hnodes, onodes, wih, who, lr)
end
InitNet (generic function with 1 method)
# Query Result
function Query(net::NetInfo, inputs::Array{Float64, 2})
# Calculate the signal entering the hidden layer
hidden_inputs = net.wih * inputs
hidden_outputs = sigmoid.(hidden_inputs)
# Calculate the signal entering the output layer
final_inputs = net.who * hidden_outputs
final_outputs = sigmoid.(final_inputs)
return final_outputs
end
Query (generic function with 1 method)
function Train!(net::NetInfo, inputs::Array{Float64, 2}, targets::Array{Float64, 2})
# PART 1: Consistent with the Query function
# Calculate the signal entering the hidden layer
hidden_inputs = net.wih * inputs
hidden_outputs = sigmoid.(hidden_inputs)
# Calculate the signal entering the output layer
final_inputs = net.who * hidden_outputs
final_outputs = sigmoid.(final_inputs)
# PART 2:Compare the resulting output with the desired output to guide the update of network weights
# Output layer error = (target - actual)
output_errors = targets - final_outputs
hidden_errors = net.who' * output_errors
net.who += net.lr .* (output_errors .* final_outputs .* (1.0 .- final_outputs)) * hidden_outputs'
net.wih += net.lr .* (hidden_errors .* hidden_outputs .* (1.0 .- hidden_outputs)) * inputs'
end
Train! (generic function with 1 method)
# parameters
input_nodes = 784
hidden_nodes = 200
output_nodes = 10
learning_rate = 0.1
epochs = 5
5
net_test = InitNet(input_nodes, input_nodes, output_nodes, learning_rate);
# import training set
training_data_file = readdlm("mnist_dataset/mnist_train.csv", ',');
# import test set
test_data_file = readdlm("mnist_dataset/mnist_test.csv", ',');
# data sets size
train_data_size = 60000
test_data_size = 10000
10000
use_gpu = true # helper to easily switch between gpu/cpu
todevice(x) = use_gpu ? gpu(x) : x
epochs = 5
# Cycle training
@time for e = 1:epochs
# Training neural network
for record = 1:train_data_size
inputs = (training_data_file[record, 2:end] ./ 255.0 .* 0.99) .+ 0.01
inputs = reshape(inputs, input_nodes, 1) # Adjustment dimension
targets = zeros(output_nodes) .+ 0.01
targets[round(Int, training_data_file[record, 1]) + 1] = 0.99
targets = reshape(targets, 10, 1)
Train!(net_test, inputs, targets)
end
end
1112.022629 seconds (11.39 M allocations: 2.729 TiB, 17.21% gc time)
# Effect test
scorecard = []
for record = 1:test_data_size
correct_label = Int(test_data_file[record, 1])
inputs = (test_data_file[record, 2:end] ./ 255.0 .* 0.99) .+ 0.01
inputs = reshape(inputs, input_nodes, 1) # Adjustment dimension
outputs = Query(net_test, inputs)
label = findmax(outputs)[2][1]
if (label - 1 == correct_label)
append!(scorecard, 1)
else
append!(scorecard, 0)
end
end
# Accuracy
print("performance = ", sum(scorecard) / length(scorecard))
performance = 0.973