Lab 1

# TensorFlow?

Tensors are represented as n-dimensional arrays of base datatypes.

• The shape of a Tensor: its number of dimensions and the size of each dimension
• The rank of a Tensor: the number of dimensions, Tensor's order or degree.
import tensorflow as tf
import mitdeeplearning as mdl
import numpy as np
import matplotlib.pyplot as plt

sport = tf.constant("Tennis", tf.string) # 0-d
numbers = tf.constant([3.141, 1.414, 2.718], tf.float64) # 1-d
matrix = tf.constnat([[1, 2, 3], [4, 5, 6]], tf.float64) # 2-d
images = tf.zeros([10, 256, 256, 3]) # 4-d

assert tf.rank(images).numpy() == 4
assert tf.shape(images).numpy().tolist() == [10, 256, 256, 3]

# slicing to access subtensors within a higher-rank Tensor

row_vector = matrix[1]        # [4. 5. 6.]
column_vector = matrix[:,2]   # [3. 6.]
scalar = matrix[1, 2]         # 6.0


# Computations on Tensors

a = tf.constant(15)
b = tf.constant(61)
c2 = a + b # tf provides overrided "+" operation
print(c1, c2)

def func(a, b):
d = tf.subtract(b, 1)
e = tf.multiply(c, d)
return e

a, b = 1.5, 2.5
e_out = func(a, b)
print(e_out) # a single scalar value


# Neural networks

\begin{aligned} y &= \sigma(Wx + b)\ W &: \text{a matrix of weights}\ b &: \text{a bias}\ x &: \text{input}\ \sigma &: \text{sigmoid activation function}\ y &: \text{output} \end{aligned}

Tensors can flow through abstract types called Layer, the building blocks of neural networks.

Implement:

• Common neural netowrk operations
• Update weights
• Compute losses
• inter-layer connectivity
class OutDenseLayer(Tf.keras.layers.Layer):
def __init__(self, n_output_nodes):
super(OurDenseLayer, self).__init__()
self.n_output_nodes = n_output_nodes

def build(self, input_shape):
d = int(input_shape[-1])

def call(self, x):
y = tf.sigmoid(z)
return y

# layer parameters are initialized randomly
# seed it to validation
tf.random.set_seed(1)
layer = OurDenseLayer(3)
layer.build((1, 2))
x_input = tf.constnat([[1, 2.]], shape=(1,2))
y = layer.call(x_input)

print(y.numpy())
mdl.test_custom_dense_layer_output(y)


Define a number of Layers using a Dense. Use Sequential model from Keras to create a single Dense layer to define the network.

from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense

n_output_nodes = 3
model = Sequential()
dense_layer = tf.keras.layers.Dense(n_output_nodes, activation='sigmoid')

x_input = tf.constant([[1, 2.]], shape=(1,2))
model_output = model(x_input).numpy()
print(model_output)


By subclassing the Model class, we can define the custom neural networks.

from tensorflow.keras import Model
from tensorflow.keras.layers import Dense

class SubclassModel(tf.keras.Model):
def __init__(self, n_output_nodes):
super(SubclassModel, self).__init__()
self.dense_layer = Dense(n_output_nodes, activation='sigmoid')
def call(self, inputs):
return self.dense_layer(inputs)

n_output_nodes = 3
model = SubclassModel(n_output_nodes)
x_input = tf.constant([1,2.], shape=(1,2))
print(model.call(x_input))

# Suppose under some instances we want out network
# to simply output the input, without any perturbation.
from tensorflow.keras import Model
from tensorflow.keras.layers import Dense

class IdentityModel(tf.keras.Model):
def __init__(self, n_output_nodes):
super(IdentityModel, self) .__init__()
self.dense_layer = Dense(n_output_nodes, activation='sigmoid')

def call(self, inputs, isidentity=False):
x = self.dense_layer(inputs)
if isidentity:
return inputs
return x

n_output_nodes = 3
model = IdentityModel(n_output_nodes)
x_input = tf.constant([1,2.], shape=(1,2))

out_activate = model.call(x_input, False)
out_identity = model.call(x_input, True)


# Automatic differentiation

Uses tf.GradientTap to trace operations for computing gradients.

All forward-pass operations get recorded to a tape. The tape played backwards when it compute the gradient. It discarded after the computation. If computing multiple gardients needed, use persistent gradient tape.

Define $y = x^2$ and compute the gradient:

x = tf.Variable(3.0)
y = x * x

assert dy_dx.numpy() == 6.0


In training neural networks, we use differentiation and stochastic gradient descent (SGD) to optimize a loss function.

Use automatic differentiation and SGD to find the minimum of the function below. We know that $x_{min} = x_f$ but uses GradientTape to solve the problem.

L = (x-x_f)^2 \ \begin{aligned} \x_f : &\text{ a variable for a desired value} \ &\text{ we are trying to optimize for}; \ L : &\text{ a loss that we are trying to minimize} \end{aligned}

x = tf.Variable([tf.random.normal([1])])
print("Initializing x = {}".format(x.numpy()))

learning_rate = 1e-2
history = []
x_f = 4

# We will run SGD for a number of iterations. At each iteration, we compute the loss,
#   compute the derivative of the loss with respect to x, and perform the SGD update.
for i in range(500):
loss = tf.pow(x - x_f, 2)

# loss minimization using gradient tape
grad = tape.gradient(loss, x) # compute the derivative of the loss with respect to x
new_x = x - learning_rate*grad # sgd update
x.assign(new_x) # update the value of x
history.append(x.numpy()[0])

plt.plot(history)
plt.plot([0, 500], [x_f, x_f])
plt.legend(('Predicted', 'True'))
plt.xlabel('Iteration')
plt.ylabel('x value')


GradientTape provides an extremely flexible framework for automatic differentiation. In order to back propagate errors through a neural network, we track forward passes on the Tape, use this information to determine the gradients, and then use these gradients for optimization using SGD.

웹페이지 색상을 선택하세요