import tensorflow as tf
data = tf.nn.batch_norm_with_global_normalization(...) 

learning_rate = 0.01 
batch_size = 100 
iterations = 1000

a_var = tf.constant(42) 
x_input = tf.placeholder(tf.float32, [None, input_size]) 
y_input = tf.placeholder(tf.float32, [None, num_classes]) 

y_pred = tf.add(tf.mul(x_input, weight_matrix), b_matrix) 

loss = tf.reduce_mean(tf.square(y_actual - y_pred)) 

with tf.Session(graph=graph) as session: 
... 
session.run(...) 
... 
Note that we can also initiate our graph with:
session = tf.Session(graph=graph) 
session.run(...) 

zero_tsr = tf.zeros([row_dim, col_dim])

ones_tsr = tf.ones([row_dim, col_dim]) 

filled_tsr = tf.fill([row_dim, col_dim], 42) 

constant_tsr = tf.constant([1,2,3])

zeros_similar = tf.zeros_like(constant_tsr) 
ones_similar = tf.ones_like(constant_tsr) 

linear_tsr = tf.linspace(start=0, stop=1, start=3) 

integer_seq_tsr = tf.range(start=6, limit=15, delta=3) 

randunif_tsr = tf.random_uniform([row_dim, col_dim], minval=0, maxval=1) 

randnorm_tsr = tf.random_normal([row_dim, col_dim], mean=0.0, stddev=1.0) 

runcnorm_tsr = tf.truncated_normal([row_dim, col_dim], mean=0.0, stddev=1.0) 

shuffled_output = tf.random_shuffle(input_tensor) 
cropped_output = tf.random_crop(input_tensor, crop_size) 

cropped_image = tf.random_crop(my_image, [height/2, width/2, 3]) 

my_var = tf.Variable(tf.zeros([row_dim, col_dim])) 

my_var = tf.Variable(tf.zeros([2,3])) 
sess = tf.Session() 
initialize_op = tf.global_variables_initializer() 
sess.run(initialize_op) 

sess = tf.Session() 
x = tf.placeholder(tf.float32, shape=[2,2]) 
y = tf.identity(x) 
x_vals = np.random.rand(2,2) 
sess.run(y, feed_dict={x: x_vals}) 
# Note that sess.run(x, feed_dict={x: x_vals}) will result in a self-referencing error. 

initializer_op = tf.global_variables_initializer() 

sess = tf.Session() 
first_var = tf.Variable(tf.zeros([2,3])) 
sess.run(first_var.initializer) 
second_var = tf.Variable(tf.zeros_like(first_var)) 
# 'second_var' depends on the 'first_var'
sess.run(second_var.initializer)

import tensorflow as tf 
sess = tf.Session() 

identity_matrix = tf.diag([1.0, 1.0, 1.0]) 
A = tf.truncated_normal([2, 3]) 
B = tf.fill([2,3], 5.0) 
C = tf.random_uniform([3,2]) 
D = tf.convert_to_tensor(np.array([[1., 2., 3.],[-3., -7., -1.],[0., 5., -2.]])) 
print(sess.run(identity_matrix)) 
[[ 1\.  0\.  0.] 
 [ 0\.  1\.  0.] 
 [ 0\.  0\.  1.]] 
print(sess.run(A)) 
[[ 0.96751703  0.11397751 -0.3438891 ] 
 [-0.10132604 -0.8432678   0.29810596]] 
print(sess.run(B)) 
[[ 5\.  5\.  5.] 
 [ 5\.  5\.  5.]] 
print(sess.run(C)) 
[[ 0.33184157  0.08907614] 
 [ 0.53189191  0.67605299] 
 [ 0.95889051 0.67061249]] 
print(sess.run(D)) 
[[ 1\.  2\.  3.] 
 [-3\. -7\. -1.] 
 [ 0\.  5\. -2.]] 

print(sess.run(A+B)) 
[[ 4.61596632  5.39771316  4.4325695 ] 
 [ 3.26702736  5.14477345  4.98265553]] 
print(sess.run(B-B)) 
[[ 0\.  0\.  0.] 
 [ 0\.  0\.  0.]] 
Multiplication 
print(sess.run(tf.matmul(B, identity_matrix))) 
[[ 5\.  5\.  5.] 
 [ 5\.  5\.  5.]] 

print(sess.run(tf.transpose(C))) 
[[ 0.67124544  0.26766731  0.99068872] 
 [ 0.25006068  0.86560275  0.58411312]] 

print(sess.run(tf.matrix_determinant(D))) 
-38.0 

print(sess.run(tf.matrix_inverse(D))) 
[[-0.5        -0.5        -0.5       ] 
 [ 0.15789474  0.05263158  0.21052632] 
 [ 0.39473684  0.13157895  0.02631579]] 

print(sess.run(tf.cholesky(identity_matrix))) 
[[ 1\.  0\.  1.] 
 [ 0\.  1\.  0.] 
 [ 0\.  0\.  1.]] 

print(sess.run(tf.self_adjoint_eig(D)) 
[[-10.65907521  -0.22750691   2.88658212] 
 [  0.21749542   0.63250104  -0.74339638] 
 [  0.84526515   0.2587998    0.46749277] 
 [ -0.4880805    0.73004459   0.47834331]] 

import tensorflow as tf 
sess = tf.Session() 

print(sess.run(tf.div(3, 4))) 
0 
print(sess.run(tf.truediv(3, 4))) 
0.75 

print(sess.run(tf.floordiv(3.0,4.0))) 
0.0 

print(sess.run(tf.mod(22.0, 5.0))) 
2.0 

print(sess.run(tf.cross([1., 0., 0.], [0., 1., 0.]))) 
[ 0\.  0\.  1.0]

# Tangent function (tan(pi/4)=1) 
print(sess.run(tf.tan(3.1416/4.)))
1.0 

def custom_polynomial(value): 
    return tf.sub(3 * tf.square(value), value) + 10
print(sess.run(custom_polynomial(11))) 
362 

import tensorflow as tf 
sess = tf.Session() 

print(sess.run(tf.nn.relu([-3., 3., 10.]))) 
[  0\.  3\.  10.] 

print(sess.run(tf.nn.relu6([-3., 3., 10.]))) 
[ 0\. 3\. 6.]

print(sess.run(tf.nn.sigmoid([-1., 0., 1.]))) 
[ 0.26894143  0.5         0.7310586 ] 

print(sess.run(tf.nn.tanh([-1., 0., 1.]))) 
[-0.76159418  0\.         0.76159418 ] 

print(sess.run(tf.nn.softsign([-1., 0., -1.]))) 
[-0.5  0\.   0.5] 

print(sess.run(tf.nn.softplus([-1., 0., -1.]))) 
[ 0.31326166  0.69314718  1.31326163] 

print(sess.run(tf.nn.elu([-1., 0., -1.]))) 
[-0.63212055  0\.          1\.        ] 

from sklearn import datasets 
iris = datasets.load_iris() 
print(len(iris.data)) 
150 
print(len(iris.target)) 
150 
print(iris.data[0]) # Sepal length, Sepal width, Petal length, Petal width 
[ 5.1 3.5 1.4 0.2] 
print(set(iris.target)) # I. setosa, I. virginica, I. versicolor 
{0, 1, 2} 

import requests
birthdata_url = 'https://github.com/nfmcclure/tensorflow_cookbook/raw/master/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat' 
birth_file = requests.get(birthdata_url) 
birth_data = birth_file.text.split('\r\n') 
birth_header = birth_data[0].split('\t') 
birth_data = [[float(x) for x in y.split('\t') if len(x)>=1] for y in birth_data[1:] if len(y)>=1]
print(len(birth_data)) 
189 
print(len(birth_data[0])) 
9

from keras.datasets import boston_housing
(x_train, y_train), (x_test, y_test) = boston_housing.load_data()
housing_header = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV'] 
print(x_train.shape[0]) 
404 
print(x_train.shape[1]) 
13 

from tensorflow.examples.tutorials.mnist import input_data 
mnist = input_data.read_data_sets("MNIST_data/"," one_hot=True) 
print(len(mnist.train.images)) 
55000 
print(len(mnist.test.images)) 
10000 
print(len(mnist.validation.images)) 
5000 
print(mnist.train.labels[1,:]) # The first label is a 3 
[ 0\.  0\.  0\.  1\.  0\.  0\.  0\.  0\.  0\.  0.] 

import requests 
import io 
from zipfile import ZipFile 
zip_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip' 
r = requests.get(zip_url) 
z = ZipFile(io.BytesIO(r.content)) 
file = z.read('SMSSpamCollection') 
text_data = file.decode() 
text_data = text_data.encode('ascii',errors='ignore') 
text_data = text_data.decode().split('\n') 
text_data = [x.split('\t') for x in text_data if len(x)>=1] 
[text_data_target, text_data_train] = [list(x) for x in zip(*text_data)] 
print(len(text_data_train)) 
5574 
print(set(text_data_target)) 
{'ham', 'spam'} 
print(text_data_train[1]) 
Ok lar... Joking wif u oni... 

import requests 
import io 
import tarfile 
movie_data_url = 'http://www.cs.cornell.edu/people/pabo/movie-review-data/rt-polaritydata.tar.gz' 
r = requests.get(movie_data_url) 
# Stream data into temp object 
stream_data = io.BytesIO(r.content) 
tmp = io.BytesIO() 
while True: 
    s = stream_data.read(16384) 
    if not s: 
        break 
    tmp.write(s) 
    stream_data.close() 
tmp.seek(0) 
# Extract tar file 
tar_file = tarfile.open(fileobj=tmp, mode="r:gz") 
pos = tar_file.extractfile('rt-polaritydata/rt-polarity.pos') 
neg = tar_file.extractfile('rt-polaritydata/rt-polarity.neg') 
# Save pos/neg reviews (Also deal with encoding) 
pos_data = [] 
for line in pos: 
    pos_data.append(line.decode('ISO-8859-1').encode('ascii',errors='ignore').decode()) 
neg_data = [] 
for line in neg: 
    neg_data.append(line.decode('ISO-8859-1').encode('ascii',errors='ignore').decode()) 
tar_file.close() 
print(len(pos_data)) 
5331 
print(len(neg_data)) 
5331 
# Print out first negative review 
print(neg_data[0]) 
simplistic , silly and tedious . 

import requests 
shakespeare_url = 'http://www.gutenberg.org/cache/epub/100/pg100.txt' 
# Get Shakespeare text 
response = requests.get(shakespeare_url) 
shakespeare_file = response.content 
# Decode binary into string 
shakespeare_text = shakespeare_file.decode('utf-8') 
# Drop first few descriptive paragraphs. 
shakespeare_text = shakespeare_text[7675:] 
print(len(shakespeare_text)) # Number of characters 
5582212

import requests 
import io 
from zipfile import ZipFile 
sentence_url = 'http://www.manythings.org/anki/deu-eng.zip' 
r = requests.get(sentence_url) 
z = ZipFile(io.BytesIO(r.content)) 
file = z.read('deu.txt') 
# Format Data 
eng_ger_data = file.decode() 
eng_ger_data = eng_ger_data.encode('ascii',errors='ignore') 
eng_ger_data = eng_ger_data.decode().split('\n') 
eng_ger_data = [x.split('\t') for x in eng_ger_data if len(x)>=1] 
[english_sentence, german_sentence] = [list(x) for x in zip(*eng_ger_data)] 
print(len(english_sentence)) 
137673 
print(len(german_sentence)) 
137673 
print(eng_ger_data[10]) 
['I' won!, 'Ich habe gewonnen!'] 

import tensorflow as tf 
sess = tf.Session() 

import numpy as np 
x_vals = np.array([1., 3., 5., 7., 9.]) 
x_data = tf.placeholder(tf.float32) 
m_const = tf.constant(3.) 
my_product = tf.multiply(x_data, m_const) 
for x_val in x_vals: 
    print(sess.run(my_product, feed_dict={x_data: x_val})) 

3.0 
9.0 
15.0 
21.0 
27.0 

import tensorflow as tf 
sess = tf.Session() 

my_array = np.array([[1., 3., 5., 7., 9.], 
                   [-2., 0., 2., 4., 6.], 
                   [-6., -3., 0., 3., 6.]]) 
x_vals = np.array([my_array, my_array + 1]) 
x_data = tf.placeholder(tf.float32, shape=(3, 5)) 

m1 = tf.constant([[1.], [0.], [-1.], [2.], [4.]]) 
m2 = tf.constant([[2.]]) 
a1 = tf.constant([[10.]]) 

prod1 = tf.matmul(x_data, m1) 
prod2 = tf.matmul(prod1, m2) 
add1 = tf.add(prod2, a1)

for x_val in x_vals: 
    print(sess.run(add1, feed_dict={x_data: x_val})) 
[[ 102.] 
 [  66.] 
 [  58.]] 
[[ 114.] 
 [  78.] 
 [  70.]] 

x_data = tf.placeholder(tf.float32, shape=(3,None))

import tensorflow as tf 
import numpy as np 
sess = tf.Session() 

x_shape = [1, 4, 4, 1] 
x_val = np.random.uniform(size=x_shape) 

x_data = tf.placeholder(tf.float32, shape=x_shape) 

my_filter = tf.constant(0.25, shape=[2, 2, 1, 1]) 
my_strides = [1, 2, 2, 1] 
mov_avg_layer= tf.nn.conv2d(x_data, my_filter, my_strides, 
                            padding='SAME', name='Moving_Avg_Window') 

    def custom_layer(input_matrix): 
        input_matrix_sqeezed = tf.squeeze(input_matrix) 
        A = tf.constant([[1., 2.], [-1., 3.]]) 
        b = tf.constant(1., shape=[2, 2]) 
        temp1 = tf.matmul(A, input_matrix_sqeezed) 
        temp = tf.add(temp1, b) # Ax + b 
        return tf.sigmoid(temp) 

with tf.name_scope('Custom_Layer') as scope: 
    custom_layer1 = custom_layer(mov_avg_layer) 

print(sess.run(custom_layer1, feed_dict={x_data: x_val})) 
[[ 0.91914582 0.96025133] 
 [ 0.87262219  0.9469803 ]] 

import matplotlib.pyplot as plt 
import tensorflow as tf 

x_vals = tf.linspace(-1., 1., 500) 
target = tf.constant(0.) 

l2_y_vals = tf.square(target - x_vals) 
l2_y_out = sess.run(l2_y_vals)

l1_y_vals = tf.abs(target - x_vals) 
l1_y_out = sess.run(l1_y_vals) 

delta1 = tf.constant(0.25) 
phuber1_y_vals = tf.multiply(tf.square(delta1), tf.sqrt(1\. +  
                        tf.square((target - x_vals)/delta1)) - 1.) 
phuber1_y_out = sess.run(phuber1_y_vals) 
delta2 = tf.constant(5.) 
phuber2_y_vals = tf.multiply(tf.square(delta2), tf.sqrt(1\. +  
                        tf.square((target - x_vals)/delta2)) - 1.) 
phuber2_y_out = sess.run(phuber2_y_vals) 

x_vals = tf.linspace(-3., 5., 500) 
target = tf.constant(1.) 
targets = tf.fill([500,], 1.)

hinge_y_vals = tf.maximum(0., 1\. - tf.multiply(target, x_vals)) 
hinge_y_out = sess.run(hinge_y_vals) 

xentropy_y_vals = - tf.multiply(target, tf.log(x_vals)) - tf.multiply((1\. - target), tf.log(1\. - x_vals)) 
xentropy_y_out = sess.run(xentropy_y_vals) 

xentropy_sigmoid_y_vals = tf.nn.sigmoid_cross_entropy_with_logits_v2(logits=x_vals, labels=targets) 
xentropy_sigmoid_y_out = sess.run(xentropy_sigmoid_y_vals) 

weight = tf.constant(0.5) 
xentropy_weighted_y_vals = tf.nn.weighted_cross_entropy_with_logits(logits=x_vals, targets=targets, pos_weight=weight) 
xentropy_weighted_y_out = sess.run(xentropy_weighted_y_vals) 

unscaled_logits = tf.constant([[1., -3., 10.]]) 
target_dist = tf.constant([[0.1, 0.02, 0.88]]) 
softmax_xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=unscaled_logits, labels=target_dist) 
print(sess.run(softmax_xentropy)) 
[ 1.16012561] 

unscaled_logits = tf.constant([[1., -3., 10.]]) 
sparse_target_dist = tf.constant([2]) 
sparse_xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=unscaled_logits, labels=sparse_target_dist) 
print(sess.run(sparse_xentropy)) 
[ 0.00012564] 

x_array = sess.run(x_vals) 
plt.plot(x_array, l2_y_out, 'b-', label='L2 Loss') 
plt.plot(x_array, l1_y_out, 'r--', label='L1 Loss') 
plt.plot(x_array, phuber1_y_out, 'k-.', label='P-Huber Loss (0.25)') 
plt.plot(x_array, phuber2_y_out, 'g:', label='P-Huber Loss (5.0)') 
plt.ylim(-0.2, 0.4) 
plt.legend(loc='lower right', prop={'size': 11}) 
plt.show() 

x_array = sess.run(x_vals) 
plt.plot(x_array, hinge_y_out, 'b-''', label='Hinge Loss''') 
plt.plot(x_array, xentropy_y_out, 'r--''', label='Cross' Entropy Loss') 
plt.plot(x_array, xentropy_sigmoid_y_out, 'k-.''', label='Cross' Entropy Sigmoid Loss') 
plt.plot(x_array, xentropy_weighted_y_out, g:''', label='Weighted' Cross Enropy Loss (x0.5)') 
plt.ylim(-1.5, 3) 
plt.legend(loc='lower right''', prop={'size''': 11}) 
plt.show() 

import numpy as np 
import tensorflow as tf 

sess = tf.Session() 

x_vals = np.random.normal(1, 0.1, 100) 
y_vals = np.repeat(10., 100) 
x_data = tf.placeholder(shape=[1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[1])) 

my_output = tf.mul(x_data, A) 

loss = tf.square(my_output - y_target)

my_opt = tf.train.GradientDescentOptimizer(learning_rate=0.02)
train_step = my_opt.minimize(loss)

init = tf.global_variable_initializer()
sess.run(init) 

for i in range(100): 
    rand_index = np.random.choice(100) 
    rand_x = [x_vals[rand_index]] 
    rand_y = [y_vals[rand_index]] 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    if (i + 1) % 25 == 0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 
        print('Loss = ' + str(sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}))) 
# Here is the output: 
Step #25 A = [ 6.23402166] 
Loss = 16.3173 
Step #50 A = [ 8.50733757] 
Loss = 3.56651 
Step #75 A = [ 9.37753201] 
Loss = 3.03149 
Step #100 A = [ 9.80041122] 
Loss = 0.0990248 

from tensorflow.python.framework import ops 
ops.reset_default_graph() 
sess = tf.Session() 

x_vals = np.concatenate((np.random.normal(-1, 1, 50), np.random.normal(3, 1, 50))) 
y_vals = np.concatenate((np.repeat(0., 50), np.repeat(1., 50))) 
x_data = tf.placeholder(shape=[1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(mean=10, shape=[1])) 

my_output = tf.add(x_data, A)

my_output_expanded = tf.expand_dims(my_output, 0) 
y_target_expanded = tf.expand_dims(y_target, 0) 

init = tf.initialize_all_variables() 
sess.run(init) 

xentropy = tf.nn.sigmoid_cross_entropy_with_logits( my_output_expanded, y_target_expanded) 

my_opt = tf.train.GradientDescentOptimizer(0.05) 
train_step = my_opt.minimize(xentropy) 

for i in range(1400): 
    rand_index = np.random.choice(100) 
    rand_x = [x_vals[rand_index]] 
    rand_y = [y_vals[rand_index]] 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    if (i + 1) % 200 == 0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 
        print('Loss = ' + str(sess.run(xentropy, feed_dict={x_data: rand_x, y_target: rand_y}))) 
Step #200 A = [ 3.59597969] 
Loss = [[ 0.00126199]] 
Step #400 A = [ 0.50947344] 
Loss = [[ 0.01149425]] 
Step #600 A = [-0.50994617] 
Loss = [[ 0.14271219]] 
Step #800 A = [-0.76606178] 
Loss = [[ 0.18807337]] 
Step #1000 A = [-0.90859312] 
Loss = [[ 0.02346182]] 
Step #1200 A = [-0.86169094] 
Loss = [[ 0.05427232]] 
Step #1400 A = [-1.08486211] 
Loss = [[ 0.04099189]] 

import matplotlib as plt 
import numpy as np 
import tensorflow as tf 
sess = tf.Session() 

batch_size = 20

x_vals = np.random.normal(1, 0.1, 100) 
y_vals = np.repeat(10., 100) 
x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[1,1])) 

my_output = tf.matmul(x_data, A) 

loss = tf.reduce_mean(tf.square(my_output - y_target)) 

my_opt = tf.train.GradientDescentOptimizer(0.02) 
train_step = my_opt.minimize(loss)
init = tf.global_variables_initializer()
sess.run(init)

loss_batch = [] 
for i in range(100): 
    rand_index = np.random.choice(100, size=batch_size) 
    rand_x = np.transpose([x_vals[rand_index]]) 
    rand_y = np.transpose([y_vals[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    if (i + 1) % 5 == 0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 
        temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
        print('Loss = ' + str(temp_loss)) 
        loss_batch.append(temp_loss) 

Step #100 A = [[ 9.86720943]] 
Loss = 0\. 

loss_stochastic = [] 
for i in range(100): 
    rand_index = np.random.choice(100) 
    rand_x = [x_vals[rand_index]] 
    rand_y = [y_vals[rand_index]] 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    if (i + 1) % 5 == 0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 
        temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
        print('Loss = ' + str(temp_loss)) 
        loss_stochastic.append(temp_loss) 

plt.plot(range(0, 100, 5), loss_stochastic, 'b-', label='Stochastic Loss') 
plt.plot(range(0, 100, 5), loss_batch, 'r--', label='Batch' Loss, size=20') 
plt.legend(loc='upper right', prop={'size': 11}) 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
from sklearn import datasets 
import tensorflow as tf 
sess = tf.Session() 

iris = datasets.load_iris() 
binary_target = np.array([1\. if x==0 else 0\. for x in iris.target]) 
iris_2d = np.array([[x[2], x[3]] for x in iris.data]) 

batch_size = 20 
x1_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
x2_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[1, 1])) 
b = tf.Variable(tf.random_normal(shape=[1, 1])) 

my_mult = tf.matmul(x2_data, A) 
my_add = tf.add(my_mult, b) 
my_output = tf.sub(x1_data, my_add) 

xentropy = tf.nn.sigmoid_cross_entropy_with_logits(my_output, y_target) 

my_opt = tf.train.GradientDescentOptimizer(0.05) 
train_step = my_opt.minimize(xentropy) 

init = tf.global_variables_initializer() 
sess.run(init) 

for i in range(1000): 
    rand_index = np.random.choice(len(iris_2d), size=batch_size) 
    rand_x = iris_2d[rand_index] 
    rand_x1 = np.array([[x[0]] for x in rand_x]) 
    rand_x2 = np.array([[x[1]] for x in rand_x]) 
    rand_y = np.array([[y] for y in binary_target[rand_index]]) 
    sess.run(train_step, feed_dict={x1_data: rand_x1, x2_data: rand_x2, y_target: rand_y}) 
    if (i + 1) % 200 == 0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ', b = ' + str(sess.run(b)))

Step #200 A = [[ 8.67285347]], b = [[-3.47147632]] 
Step #400 A = [[ 10.25393486]], b = [[-4.62928772]] 
Step #600 A = [[ 11.152668]], b = [[-5.4077611]] 
Step #800 A = [[ 11.81016064]], b = [[-5.96689034]] 
Step #1000 A = [[ 12.41202831]], b = [[-6.34769201]] 

[[slope]] = sess.run(A) 
[[intercept]] = sess.run(b) 
x = np.linspace(0, 3, num=50) 
ablineValues = [] 
for i in x: 
    ablineValues.append(slope*i+intercept) 

setosa_x = [a[1] for i,a in enumerate(iris_2d) if binary_target[i]==1] 
setosa_y = [a[0] for i,a in enumerate(iris_2d) if binary_target[i]==1] 
non_setosa_x = [a[1] for i,a in enumerate(iris_2d) if binary_target[i]==0] 
non_setosa_y = [a[0] for i,a in enumerate(iris_2d) if binary_target[i]==0] 
plt.plot(setosa_x, setosa_y, 'rx', ms=10, mew=2, label='setosa') 
plt.plot(non_setosa_x, non_setosa_y, 'ro', label='Non-setosa') 
plt.plot(x, ablineValues, 'b-') 
plt.xlim([0.0, 2.7]) 
plt.ylim([0.0, 7.1]) 
plt.suptitle('Linear' Separator For I.setosa', fontsize=20) 
plt.xlabel('Petal Length') 
plt.ylabel('Petal Width') 
plt.legend(loc='lower right') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
sess = tf.Session() 
x_vals = np.random.normal(1, 0.1, 100) 
y_vals = np.repeat(10., 100) 
x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
batch_size = 25 
train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 
x_vals_train = x_vals[train_indices] 
x_vals_test = x_vals[test_indices] 
y_vals_train = y_vals[train_indices] 
y_vals_test = y_vals[test_indices] 
A = tf.Variable(tf.random_normal(shape=[1,1])) 

my_output = tf.matmul(x_data, A) 
loss = tf.reduce_mean(tf.square(my_output - y_target))
my_opt = tf.train.GradientDescentOptimizer(0.02)
train_step = my_opt.minimize(loss)
init = tf.global_variables_initializer() 
sess.run(init) 

for i in range(100): 
    rand_index = np.random.choice(len(x_vals_train), size=batch_size) rand_x = np.transpose([x_vals_train[rand_index]]) 
    rand_y = np.transpose([y_vals_train[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    if (i + 1) % 25 == 0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 
        print('Loss = ' + str(sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}))) 
Step #25 A = [[ 6.39879179]] 
Loss = 13.7903 
Step #50 A = [[ 8.64770794]] 
Loss = 2.53685 
Step #75 A = [[ 9.40029907]] 
Loss = 0.818259 
Step #100 A = [[ 9.6809473]] 
Loss = 1.10908 

mse_test = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_test]), y_target: np.transpose([y_vals_test])}) 
mse_train = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_train]), y_target: np.transpose([y_vals_train])}) 
print('MSE' on test:' + str(np.round(mse_test, 2))) 
print('MSE' on train:' + str(np.round(mse_train, 2))) 
MSE on test:1.35 
MSE on train:0.88 

from tensorflow.python.framework import ops 
ops.reset_default_graph() 
sess = tf.Session() 
batch_size = 25 
x_vals = np.concatenate((np.random.normal(-1, 1, 50), np.random.normal(2, 1, 50))) 
y_vals = np.concatenate((np.repeat(0., 50), np.repeat(1., 50))) 
x_data = tf.placeholder(shape=[1, None], dtype=tf.float32) 
y_target = tf.placeholder(shape=[1, None], dtype=tf.float32) 
train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 
x_vals_train = x_vals[train_indices] 
x_vals_test = x_vals[test_indices] 
y_vals_train = y_vals[train_indices] 
y_vals_test = y_vals[test_indices] 
A = tf.Variable(tf.random_normal(mean=10, shape=[1])) 

my_output = tf.add(x_data, A) 
init = tf.initialize_all_variables() 
sess.run(init) 
xentropy = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(my_output, y_target)) 
my_opt = tf.train.GradientDescentOptimizer(0.05) 
train_step = my_opt.minimize(xentropy) 

for i in range(1800): 
    rand_index = np.random.choice(len(x_vals_train), size=batch_size) 
    rand_x = [x_vals_train[rand_index]] 
    rand_y = [y_vals_train[rand_index]] 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    if (i+1)%200==0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 
        print('Loss = ' + str(sess.run(xentropy, feed_dict={x_data: rand_x, y_target: rand_y}))) 
Step #200 A = [ 6.64970636] 
Loss = 3.39434 
Step #400 A = [ 2.2884655] 
Loss = 0.456173 
Step #600 A = [ 0.29109824] 
Loss = 0.312162 
Step #800 A = [-0.20045301] 
Loss = 0.241349 
Step #1000 A = [-0.33634067] 
Loss = 0.376786 
Step #1200 A = [-0.36866501] 
Loss = 0.271654 
Step #1400 A = [-0.3727718] 
Loss = 0.294866 
Step #1600 A = [-0.39153299] 
Loss = 0.202275 
Step #1800 A = [-0.36630616] 
Loss = 0.358463 

y_prediction = tf.squeeze(tf.round(tf.nn.sigmoid(tf.add(x_data, A)))) 
correct_prediction = tf.equal(y_prediction, y_target) 
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 
acc_value_test = sess.run(accuracy, feed_dict={x_data: [x_vals_test], y_target: [y_vals_test]}) 
acc_value_train = sess.run(accuracy, feed_dict={x_data: [x_vals_train], y_target: [y_vals_train]}) 
print('Accuracy' on train set: ' + str(acc_value_train)) 
print('Accuracy' on test set: ' + str(acc_value_test)) 
Accuracy on train set: 0.925 
Accuracy on test set: 0.95 

A_result = sess.run(A) 
bins = np.linspace(-5, 5, 50) 
plt.hist(x_vals[0:50], bins, alpha=0.5, label='N(-1,1)', color='white') 
plt.hist(x_vals[50:100], bins[0:50], alpha=0.5, label='N(2,1)', color='red') 
plt.plot((A_result, A_result), (0, 8), 'k--', linewidth=3, label='A = '+ str(np.round(A_result, 2))) 
plt.legend(loc='upper right') 
plt.title('Binary Classifier, Accuracy=' + str(np.round(acc_value, 2))) 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
sess = tf.Session() 
x_vals = np.linspace(0, 10, 100) 
y_vals = x_vals + np.random.normal(0, 1, 100) 

x_vals_column = np.transpose(np.matrix(x_vals)) 
ones_column = np.transpose(np.matrix(np.repeat(1, 100))) 
A = np.column_stack((x_vals_column, ones_column)) 
b = np.transpose(np.matrix(y_vals)) 

A_tensor = tf.constant(A) 
b_tensor = tf.constant(b)

tA_A = tf.matmul(tf.transpose(A_tensor), A_tensor) 
tA_A_inv = tf.matrix_inverse(tA_A) 
product = tf.matmul(tA_A_inv, tf.transpose(A_tensor)) 
solution = tf.matmul(product, b_tensor) 
solution_eval = sess.run(solution) 

slope = solution_eval[0][0] 
y_intercept = solution_eval[1][0] 
print('slope: ' + str(slope)) 
print('y_intercept: ' + str(y_intercept)) 
slope: 0.955707151739 
y_intercept: 0.174366829314 
best_fit = [] 
for i in x_vals: 
    best_fit.append(slope*i+y_intercept) 

plt.plot(x_vals, y_vals, 'o', label='Data') 
plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 
plt.legend(loc='upper left') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from tensorflow.python.framework import ops 
ops.reset_default_graph() 
sess = tf.Session() 
x_vals = np.linspace(0, 10, 100) 
y_vals = x_vals + np.random.normal(0, 1, 100) 
x_vals_column = np.transpose(np.matrix(x_vals)) 
ones_column = np.transpose(np.matrix(np.repeat(1, 100))) 
A = np.column_stack((x_vals_column, ones_column)) 
b = np.transpose(np.matrix(y_vals)) 
A_tensor = tf.constant(A) 
b_tensor = tf.constant(b) 

tA_A = tf.matmul(tf.transpose(A_tensor), A_tensor) 
L = tf.cholesky(tA_A) 
tA_b = tf.matmul(tf.transpose(A_tensor), b) 
sol1 = tf.matrix_solve(L, tA_b) 
sol2 = tf.matrix_solve(tf.transpose(L), sol1) 

solution_eval = sess.run(sol2) 
slope = solution_eval[0][0] 
y_intercept = solution_eval[1][0] 
print('slope: ' + str(slope)) 
print('y_intercept: ' + str(y_intercept)) 
slope: 0.956117676145 
y_intercept: 0.136575513864 
best_fit = [] 
for i in x_vals: 
    best_fit.append(slope*i+y_intercept) 
plt.plot(x_vals, y_vals, 'o', label='Data') 
plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 
plt.legend(loc='upper left') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
from tensorflow.python.framework import ops 
ops.reset_default_graph() 
sess = tf.Session() 
iris = datasets.load_iris() 
x_vals = np.array([x[3] for x in iris.data]) 
y_vals = np.array([y[0] for y in iris.data])

learning_rate = 0.05 
batch_size = 25 
x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[1,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 

model_output = tf.add(tf.matmul(x_data, A), b)

loss = tf.reduce_mean(tf.square(y_target - model_output)) 
init = tf.global_variables_initializer() 
sess.run(init) 
my_opt = tf.train.GradientDescentOptimizer(learning_rate) 
train_step = my_opt.minimize(loss)

loss_vec = [] 
for i in range(100): 
    rand_index = np.random.choice(len(x_vals), size=batch_size) 
    rand_x = np.transpose([x_vals[rand_index]]) 
    rand_y = np.transpose([y_vals[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss) 
    if (i+1)%25==0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 
        print('Loss = ' + str(temp_loss)) 

Step #25 A = [[ 2.17270374]] b = [[ 2.85338426]] 
Loss = 1.08116 
Step #50 A = [[ 1.70683455]] b = [[ 3.59916329]] 
Loss = 0.796941 
Step #75 A = [[ 1.32762754]] b = [[ 4.08189011]] 
Loss = 0.466912 
Step #100 A = [[ 1.15968263]] b = [[ 4.38497639]] 
Loss = 0.281003

[slope] = sess.run(A) 
[y_intercept] = sess.run(b) 
best_fit = [] 
for i in x_vals: 
    best_fit.append(slope*i+y_intercept)

plt.plot(x_vals, y_vals, 'o', label='Data Points') 
plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 
plt.legend(loc='upper left') 
plt.title('Sepal Length vs Petal Width') 
plt.xlabel('Petal Width') 
plt.ylabel('Sepal Length') 
plt.show() 
plt.plot(loss_vec, 'k-') 
plt.title('L2 Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('L2 Loss') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
sess = tf.Session() 
iris = datasets.load_iris() 
x_vals = np.array([x[3] for x in iris.data]) 
y_vals = np.array([y[0] for y in iris.data]) 
batch_size = 25 
learning_rate = 0.1 # Will not converge with learning rate at 0.4 
iterations = 50 
x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[1,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 
model_output = tf.add(tf.matmul(x_data, A), b)

loss_l1 = tf.reduce_mean(tf.abs(y_target - model_output)) 

init = tf.global_variables_initializer() 
sess.run(init) 
my_opt_l1 = tf.train.GradientDescentOptimizer(learning_rate) 
train_step_l1 = my_opt_l1.minimize(loss_l1) 
loss_vec_l1 = [] 
for i in range(iterations): 
    rand_index = np.random.choice(len(x_vals), size=batch_size) 
    rand_x = np.transpose([x_vals[rand_index]]) 
    rand_y = np.transpose([y_vals[rand_index]]) 
    sess.run(train_step_l1, feed_dict={x_data: rand_x, y_target: rand_y}) 
    temp_loss_l1 = sess.run(loss_l1, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec_l1.append(temp_loss_l1) 
    if (i+1)%25==0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 

plt.plot(loss_vec_l1, 'k-', label='L1 Loss') 
plt.plot(loss_vec_l2, 'r--', label='L2 Loss') 
plt.title('L1 and L2 Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('L1 Loss') 
plt.legend(loc='upper right') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
sess = tf.Session() 
iris = datasets.load_iris() 
x_vals = np.array([x[3] for x in iris.data]) 
y_vals = np.array([y[0] for y in iris.data]) 
batch_size = 50 
x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[1,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 
model_output = tf.add(tf.matmul(x_data, A), b)

deming_numerator = tf.abs(tf.sub(y_target, tf.add(tf.matmul(x_data, A), b))) 
deming_denominator = tf.sqrt(tf.add(tf.square(A),1)) 
loss = tf.reduce_mean(tf.truediv(deming_numerator, deming_denominator)) 

init = tf.global_variables_initializer() 
sess.run(init) 
my_opt = tf.train.GradientDescentOptimizer(0.1) 
train_step = my_opt.minimize(loss) 
loss_vec = [] 
for i in range(250): 
    rand_index = np.random.choice(len(x_vals), size=batch_size) 
    rand_x = np.transpose([x_vals[rand_index]]) 
    rand_y = np.transpose([y_vals[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss) 
    if (i+1)%50==0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 
        print('Loss = ' + str(temp_loss)) 

[slope] = sess.run(A) 
[y_intercept] = sess.run(b) 
best_fit = [] 
for i in x_vals: 
    best_fit.append(slope*i+y_intercept)

plt.plot(x_vals, y_vals, 'o', label='Data Points') 
plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 
plt.legend(loc='upper left') 
plt.title('Sepal Length vs petal Width') 
plt.xlabel('petal Width') 
plt.ylabel('Sepal Length') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
from tensorflow.python.framework import ops 
ops.reset_default_graph() 
sess = tf.Session() 
iris = datasets.load_iris() 
x_vals = np.array([x[3] for x in iris.data]) 
y_vals = np.array([y[0] for y in iris.data]) 
batch_size = 50 
learning_rate = 0.001 
x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[1,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 
model_output = tf.add(tf.matmul(x_data, A), b)

lasso_param = tf.constant(0.9) 
heavyside_step = tf.truediv(1., tf.add(1., tf.exp(tf.multiply(-100., tf.subtract(A, lasso_param))))) 
regularization_param = tf.mul(heavyside_step, 99.) 
loss = tf.add(tf.reduce_mean(tf.square(y_target - model_output)), regularization_param) 

init = tf.global_variables_initializer() 
sess.run(init) 
my_opt = tf.train.GradientDescentOptimizer(learning_rate) 
train_step = my_opt.minimize(loss) 

loss_vec = [] 
for i in range(1500): 
    rand_index = np.random.choice(len(x_vals), size=batch_size) 
    rand_x = np.transpose([x_vals[rand_index]]) 
    rand_y = np.transpose([y_vals[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss[0]) 
    if (i+1)%300==0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 
        print('Loss = ' + str(temp_loss)) 

Step #300 A = [[ 0.82512331]] b = [[ 2.30319238]] 
Loss = [[ 6.84168959]] 
Step #600 A = [[ 0.8200165]] b = [[ 3.45292258]] 
Loss = [[ 2.02759886]] 
Step #900 A = [[ 0.81428504]] b = [[ 4.08901262]] 
Loss = [[ 0.49081498]] 
Step #1200 A = [[ 0.80919558]] b = [[ 4.43668795]] 
Loss = [[ 0.40478843]] 
Step #1500 A = [[ 0.80433637]] b = [[ 4.6360755]] 
Loss = [[ 0.23839757]] 

ridge_param = tf.constant(1.) 
ridge_loss = tf.reduce_mean(tf.square(A)) 
loss = tf.expand_dims(tf.add(tf.reduce_mean(tf.square(y_target - model_output)), tf.multiply(ridge_param, ridge_loss)), 0) 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
sess = tf.Session()

iris = datasets.load_iris() 
x_vals = np.array([[x[1], x[2], x[3]] for x in iris.data]) 
y_vals = np.array([y[0] for y in iris.data]) 

batch_size = 50 
learning_rate = 0.001 
x_data = tf.placeholder(shape=[None, 3], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[3,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 
model_output = tf.add(tf.matmul(x_data, A), b) 

elastic_param1 = tf.constant(1.) 
elastic_param2 = tf.constant(1.) 
l1_a_loss = tf.reduce_mean(tf.abs(A)) 
l2_a_loss = tf.reduce_mean(tf.square(A)) 
e1_term = tf.multiply(elastic_param1, l1_a_loss) 
e2_term = tf.multiply(elastic_param2, l2_a_loss) 
loss = tf.expand_dims(tf.add(tf.add(tf.reduce_mean(tf.square(y_target - model_output)), e1_term), e2_term), 0)

init = tf.global_variables_initializer() 
sess.run(init) 
my_opt = tf.train.GradientDescentOptimizer(learning_rate) 
train_step = my_opt.minimize(loss) 
loss_vec = [] 
for i in range(1000): 
    rand_index = np.random.choice(len(x_vals), size=batch_size) 
    rand_x = x_vals[rand_index] 
    rand_y = np.transpose([y_vals[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss[0]) 
    if (i+1)%250==0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 
        print('Loss = ' + str(temp_loss)) 

Step #250 A = [[ 0.42095602] 
 [ 0.1055888 ] 
 [ 1.77064979]] b = [[ 1.76164341]] 
Loss = [ 2.87764359] 
Step #500 A = [[ 0.62762028] 
 [ 0.06065864] 
 [ 1.36294949]] b = [[ 1.87629771]] 
Loss = [ 1.8032167] 
Step #750 A = [[ 0.67953539] 
 [ 0.102514 ] 
 [ 1.06914485]] b = [[ 1.95604002]] 
Loss = [ 1.33256555] 
Step #1000 A = [[ 0.6777274 ] 
 [ 0.16535147] 
 [ 0.8403284 ]] b = [[ 2.02246833]] 
Loss = [ 1.21458709]

plt.plot(loss_vec, 'k-') 
plt.title('Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('Loss') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
import requests 
from sklearn import datasets 
from sklearn.preprocessing import normalize 
from tensorflow.python.framework import ops 
ops.reset_default_graph() 
sess = tf.Session()

birth_weight_file = 'birth_weight.csv'
# Download data and create data file if file does not exist in current directory
if not os.path.exists(birth_weight_file):
    birthdata_url = 'https://github.com/nfmcclure/tensorflow_cookbook/raw/master/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat'
    birth_file = requests.get(birthdata_url)
    birth_data = birth_file.text.split('\r\n')
    birth_header = birth_data[0].split('\t')
    birth_data = [[float(x) for x in y.split('\t') if len(x)>=1] for y in birth_data[1:] if len(y)>=1]
    with open(birth_weight_file, 'w', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(birth_header)
        writer.writerows(birth_data)

# Read birth weight data into memory
birth_data = []
with open(birth_weight_file, newline='') as csvfile:
    csv_reader = csv.reader(csvfile)
    birth_header = next(csv_reader)
    for row in csv_reader:
        birth_data.append(row)
    birth_data = [[float(x) for x in row] for row in birth_data]
# Pull out target variable
y_vals = np.array([x[0] for x in birth_data])
# Pull out predictor variables (not id, not target, and not birthweight)
x_vals = np.array([x[1:8] for x in birth_data])

train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 
x_vals_train = x_vals[train_indices] 
x_vals_test = x_vals[test_indices] 
y_vals_train = y_vals[train_indices] 
y_vals_test = y_vals[test_indices]

def normalize_cols(m, col_min=np.array([None]), col_max=np.array([None])):
    if not col_min[0]:
        col_min = m.min(axis=0)
    if not col_max[0]:
        col_max = m.max(axis=0)
    return (m-col_min) / (col_max - col_min), col_min, col_max

x_vals_train, train_min, train_max = np.nan_to_num(normalize_cols(x_vals_train))
x_vals_test = np.nan_to_num(normalize_cols(x_vals_test, train_min, train_max))

batch_size = 25 
x_data = tf.placeholder(shape=[None, 7], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
A = tf.Variable(tf.random_normal(shape=[7,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 
model_output = tf.add(tf.matmul(x_data, A), b)

loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(model_output, y_target)) 
init = tf.global_variables_initializer() 
sess.run(init) 
my_opt = tf.train.GradientDescentOptimizer(0.01) 
train_step = my_opt.minimize(loss)

prediction = tf.round(tf.sigmoid(model_output)) 
predictions_correct = tf.cast(tf.equal(prediction, y_target), tf.float32) 
accuracy = tf.reduce_mean(predictions_correct)

loss_vec = [] 
train_acc = [] 
test_acc = [] 
for i in range(1500): 
    rand_index = np.random.choice(len(x_vals_train), size=batch_size) 
    rand_x = x_vals_train[rand_index] 
    rand_y = np.transpose([y_vals_train[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 
    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss) 
    temp_acc_train = sess.run(accuracy, feed_dict={x_data: x_vals_train, y_target: np.transpose([y_vals_train])}) 
    train_acc.append(temp_acc_train) 
    temp_acc_test = sess.run(accuracy, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])}) 
    test_acc.append(temp_acc_test)

plt.plot(loss_vec, 'k-') 
plt.title('Cross' Entropy Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('Cross' Entropy Loss') 
plt.show() 
plt.plot(train_acc, 'k-', label='Train Set Accuracy') 
plt.plot(test_acc, 'r--', label='Test Set Accuracy') 
plt.title('Train and Test Accuracy') 
plt.xlabel('Generation') 
plt.ylabel('Accuracy') 
plt.legend(loc='lower right') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 

sess = tf.Session() 
iris = datasets.load_iris() 
x_vals = np.array([[x[0], x[3]] for x in iris.data]) 
y_vals = np.array([1 if y==0 else -1 for y in iris.target])

train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 
x_vals_train = x_vals[train_indices] 
x_vals_test = x_vals[test_indices] 
y_vals_train = y_vals[train_indices] 
y_vals_test = y_vals[test_indices] 

batch_size = 100 

x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 

A = tf.Variable(tf.random_normal(shape=[2,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 

model_output = tf.subtract(tf.matmul(x_data, A), b) 

l2_norm = tf.reduce_sum(tf.square(A)) 
alpha = tf.constant([0.1]) 
classification_term = tf.reduce_mean(tf.maximum(0., tf.subtract(1., tf.multiply(model_output, y_target)))) 

loss = tf.add(classification _term, tf.multiply(alpha, l2_norm)) 

prediction = tf.sign(model_output) 
accuracy = tf.reduce_mean(tf.cast(tf.equal(prediction, y_target), tf.float32)) 

my_opt = tf.train.GradientDescentOptimizer(0.01) 
train_step = my_opt.minimize(loss) 

init = tf.global_variables_initializer() 
sess.run(init) 

loss_vec = [] 
train_accuracy = [] 
test_accuracy = [] 
for i in range(500): 
   rand_index = np.random.choice(len(x_vals_train), size=batch_size) 
   rand_x = x_vals_train[rand_index] 
    rand_y = np.transpose([y_vals_train[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 

    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss) 

    train_acc_temp = sess.run(accuracy, feed_dict={x_data: x_vals_train, y_target: np.transpose([y_vals_train])}) 
    train_accuracy.append(train_acc_temp) 

    test_acc_temp = sess.run(accuracy, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])}) 
    test_accuracy.append(test_acc_temp) 

    if (i+1)%100==0: 
        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 
        print('Loss = ' + str(temp_loss))

Step #100 A = [[-0.10763293] 
 [-0.65735245]] b = [[-0.68752676]] 
Loss = [ 0.48756418] 
Step #200 A = [[-0.0650763 ] 
 [-0.89443302]] b = [[-0.73912662]] 
Loss = [ 0.38910741] 
Step #300 A = [[-0.02090022] 
 [-1.12334013]] b = [[-0.79332656]] 
Loss = [ 0.28621092] 
Step #400 A = [[ 0.03189624] 
 [-1.34912157]] b = [[-0.8507266]] 
Loss = [ 0.22397576] 
Step #500 A = [[ 0.05958777] 
 [-1.55989814]] b = [[-0.9000265]] 
Loss = [ 0.20492229] 

[[a1], [a2]] = sess.run(A) 
[[b]] = sess.run(b) 
slope = -a2/a1 
y_intercept = b/a1 

x1_vals = [d[1] for d in x_vals] 

best_fit = [] 
for i in x1_vals: 
    best_fit.append(slope*i+y_intercept) 

setosa_x = [d[1] for i,d in enumerate(x_vals) if y_vals[i]==1] 
setosa_y = [d[0] for i,d in enumerate(x_vals) if y_vals[i]==1] 
not_setosa_x = [d[1] for i,d in enumerate(x_vals) if y_vals[i]==-1] 
not_setosa_y = [d[0] for i,d in enumerate(x_vals) if y_vals[i]==-1] 

plt.plot(setosa_x, setosa_y, 'o', label='I. setosa') 
plt.plot(not_setosa_x, not_setosa_y, 'x', label='Non-setosa') 
plt.plot(x1_vals, best_fit, 'r-', label='Linear Separator', linewidth=3) 
plt.ylim([0, 10]) 
plt.legend(loc='lower right') 
plt.title('Sepal Length vs Petal Width') 
plt.xlabel('Petal Width') 
plt.ylabel('Sepal Length') 
plt.show() 

plt.plot(train_accuracy, 'k-', label='Training Accuracy') 
plt.plot(test_accuracy, 'r--', label='Test Accuracy') 
plt.title('Train and Test Set Accuracies') 
plt.xlabel('Generation') 
plt.ylabel('Accuracy') 
plt.legend(loc='lower right') 
plt.show() 

plt.plot(loss_vec, 'k-') 
plt.title('Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('Loss') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
sess = tf.Session() 
iris = datasets.load_iris() 
x_vals = np.array([x[3] for x in iris.data]) 
y_vals = np.array([y[0] for y in iris.data]) 
train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 
x_vals_train = x_vals[train_indices] 
x_vals_test = x_vals[test_indices] 
y_vals_train = y_vals[train_indices] 
y_vals_test = y_vals[test_indices]

batch_size = 50 

x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 

A = tf.Variable(tf.random_normal(shape=[1,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 

model_output = tf.add(tf.matmul(x_data, A), b) 

epsilon = tf.constant([0.5]) 
loss = tf.reduce_mean(tf.maximum(0., tf.subtract(tf.abs(tf.subtract(model_output, y_target)), epsilon))) 

my_opt = tf.train.GradientDescentOptimizer(0.075) 
train_step = my_opt.minimize(loss) 

init = tf.global_variables_initializer() 
sess.run(init) 

train_loss = [] 
test_loss = [] 
for i in range(200): 
    rand_index = np.random.choice(len(x_vals_train), size=batch_size) 
    rand_x = np.transpose([x_vals_train[rand_index]]) 
    rand_y = np.transpose([y_vals_train[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 

    temp_train_loss = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_train]), y_target: np.transpose([y_vals_train])}) 
    train_loss.append(temp_train_loss) 

    temp_test_loss = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_test]), y_target: np.transpose([y_vals_test])}) 
    test_loss.append(temp_test_loss) 
    if (i+1)%50==0: 
        print('-----------') 
        print('Generation: ' + str(i)) 
        print('A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 
        print('Train Loss = ' + str(temp_train_loss)) 
        print('Test Loss = ' + str(temp_test_loss)) 

Generation: 50 
A = [[ 2.20651722]] b = [[ 2.71290684]] 
Train Loss = 0.609453 
Test Loss = 0.460152 
----------- 
Generation: 100 
A = [[ 1.6440177]] b = [[ 3.75240564]] 
Train Loss = 0.242519 
Test Loss = 0.208901 
----------- 
Generation: 150 
A = [[ 1.27711761]] b = [[ 4.3149066]] 
Train Loss = 0.108192 
Test Loss = 0.119284 
----------- 
Generation: 200 
A = [[ 1.05271816]] b = [[ 4.53690529]] 
Train Loss = 0.0799957 
Test Loss = 0.107551 

[[slope]] = sess.run(A) 
[[y_intercept]] = sess.run(b) 
[width] = sess.run(epsilon) 

best_fit = [] 
best_fit_upper = [] 
best_fit_lower = [] 
for i in x_vals: 
  best_fit.append(slope*i+y_intercept) 
  best_fit_upper.append(slope*i+y_intercept+width) 
  best_fit_lower.append(slope*i+y_intercept-width) 

plt.plot(x_vals, y_vals, 'o', label='Data Points') 
plt.plot(x_vals, best_fit, 'r-', label='SVM Regression Line', linewidth=3) 
plt.plot(x_vals, best_fit_upper, 'r--', linewidth=2) 
plt.plot(x_vals, best_fit_lower, 'r--', linewidth=2) 
plt.ylim([0, 10]) 
plt.legend(loc='lower right') 
plt.title('Sepal Length vs Petal Width') 
plt.xlabel('Petal Width') 
plt.ylabel('Sepal Length') 
plt.show() 
plt.plot(train_loss, 'k-', label='Train Set Loss') 
plt.plot(test_loss, 'r--', label='Test Set Loss') 
plt.title('L2 Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('L2 Loss') 
plt.legend(loc='upper right') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
sess = tf.Session() 

(x_vals, y_vals) = datasets.make_circles(n_samples=500, factor=.5, noise=.1) 
y_vals = np.array([1 if y==1 else -1 for y in y_vals]) 
class1_x = [x[0] for i,x in enumerate(x_vals) if y_vals[i]==1] 
class1_y = [x[1] for i,x in enumerate(x_vals) if y_vals[i]==1] 
class2_x = [x[0] for i,x in enumerate(x_vals) if y_vals[i]==-1] 
class2_y = [x[1] for i,x in enumerate(x_vals) if y_vals[i]==-1]

batch_size = 250 
x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
prediction_grid = tf.placeholder(shape=[None, 2], dtype=tf.float32) 
b = tf.Variable(tf.random_normal(shape=[1,batch_size])) 

gamma = tf.constant(-50.0) 
dist = tf.reduce_sum(tf.square(x_data), 1) 
dist = tf.reshape(dist, [-1,1]) 
sq_dists = tf.add(tf.subtract(dist, tf.multiply(2., tf.matmul(x_data, tf.transpose(x_data)))), tf.transpose(dist)) 
my_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists)))  

model_output = tf.matmul(b, my_kernel) 
first_term = tf.reduce_sum(b) 
b_vec_cross = tf.matmul(tf.transpose(b), b) 
y_target_cross = tf.matmul(y_target, tf.transpose(y_target)) 
second_term = tf.reduce_sum(tf.multiply(my_kernel, tf.multiply(b_vec_cross, y_target_cross))) 
loss = tf.negative(tf.subtract(first_term, second_term))

rA = tf.reshape(tf.reduce_sum(tf.square(x_data), 1),[-1,1]) 
rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1),[-1,1]) 
pred_sq_dist = tf.add(tf.subtract(rA, tf.multiply(2., tf.matmul(x_data, tf.transpose(prediction_grid)))), tf.transpose(rB)) 
pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist))) 

prediction_output = tf.matmul(tf.multiply(tf.transpose(y_target),b), pred_kernel) 
prediction = tf.sign(prediction_output-tf.reduce_mean(prediction_output)) 
accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.squeeze(prediction), tf.squeeze(y_target)), tf.float32)) 

my_opt = tf.train.GradientDescentOptimizer(0.001) 
train_step = my_opt.minimize(loss) 
init = tf.global_variables_initializer() 
sess.run(init) 

loss_vec = [] 
batch_accuracy = [] 
for i in range(500): 
    rand_index = np.random.choice(len(x_vals), size=batch_size) 
    rand_x = x_vals[rand_index] 
    rand_y = np.transpose([y_vals[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 

    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss) 

    acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x, 
                                             y_target: rand_y, 
                                             prediction_grid:rand_x}) 
    batch_accuracy.append(acc_temp) 

    if (i+1)%100==0: 
        print('Step #' + str(i+1)) 
        print('Loss = ' + str(temp_loss)) 

Step #100 
Loss = -28.0772 
Step #200 
Loss = -3.3628 
Step #300 
Loss = -58.862 
Step #400 
Loss = -75.1121 
Step #500 
Loss = -84.8905 

x_min, x_max = x_vals[:, 0].min() - 1, x_vals[:, 0].max() + 1 
y_min, y_max = x_vals[:, 1].min() - 1, x_vals[:, 1].max() + 1 
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02), 
                     np.arange(y_min, y_max, 0.02)) 
grid_points = np.c_[xx.ravel(), yy.ravel()] 
[grid_predictions] = sess.run(prediction, feed_dict={x_data: x_vals, 
                                                   y_target: np.transpose([y_vals]), 
                                                   prediction_grid: grid_points}) 
grid_predictions = grid_predictions.reshape(xx.shape) 

plt.contourf(xx, yy, grid_predictions, cmap=plt.cm.Paired, alpha=0.8) 
plt.plot(class1_x, class1_y, 'ro', label='Class 1') 
plt.plot(class2_x, class2_y, 'kx', label='Class -1') 
plt.legend(loc='lower right') 
plt.ylim([-1.5, 1.5]) 
plt.xlim([-1.5, 1.5]) 
plt.show() 

plt.plot(batch_accuracy, 'k-', label='Accuracy') 
plt.title('Batch Accuracy') 
plt.xlabel('Generation') 
plt.ylabel('Accuracy') 
plt.legend(loc='lower right') 
plt.show() 

plt.plot(loss_vec, 'k-') 
plt.title('Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('Loss') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
sess = tf.Session() 

iris = datasets.load_iris() 
x_vals = np.array([[x[0], x[3]] for x in iris.data]) 
y_vals = np.array([1 if y==0 else -1 for y in iris.target]) 
class1_x = [x[0] for i,x in enumerate(x_vals) if y_vals[i]==1] 
class1_y = [x[1] for i,x in enumerate(x_vals) if y_vals[i]==1] 
class2_x = [x[0] for i,x in enumerate(x_vals) if y_vals[i]==-1] 
class2_y = [x[1] for i,x in enumerate(x_vals) if y_vals[i]==-1] 

batch_size = 100 

x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32) 
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
prediction_grid = tf.placeholder(shape=[None, 2], dtype=tf.float32) 

b = tf.Variable(tf.random_normal(shape=[1,batch_size]))

gamma = tf.constant(-10.0) 
dist = tf.reduce_sum(tf.square(x_data), 1) 
dist = tf.reshape(dist, [-1,1]) 
sq_dists = tf.add(tf.subtract(dist, tf.multiply(2., tf.matmul(x_data, tf.transpose(x_data)))), tf.transpose(dist)) 
my_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists))) 
# We now compute the loss for the dual optimization problem, as follows: 
model_output = tf.matmul(b, my_kernel) 
first_term = tf.reduce_sum(b) 
b_vec_cross = tf.matmul(tf.transpose(b), b) 
y_target_cross = tf.matmul(y_target, tf.transpose(y_target)) 
second_term = tf.reduce_sum(tf.multiply(my_kernel, tf.multiply(b_vec_cross, y_target_cross))) 
loss = tf.negative(tf.subtract(first_term, second_term)) 

rA = tf.reshape(tf.reduce_sum(tf.square(x_data), 1),[-1,1]) 
rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1),[-1,1]) 
pred_sq_dist = tf.add(tf.subtract(rA, tf.mul(2., tf.matmul(x_data, tf.transpose(prediction_grid)))), tf.transpose(rB)) 
pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist))) 

prediction_output = tf.matmul(tf.multiply(tf.transpose(y_target),b), pred_kernel) 
prediction = tf.sign(prediction_output-tf.reduce_mean(prediction_output)) 
accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.squeeze(prediction), tf.squeeze(y_target)), tf.float32)) 

my_opt = tf.train.GradientDescentOptimizer(0.01) 
train_step = my_opt.minimize(loss) 
init = tf.initialize_all_variables() 
sess.run(init)

loss_vec = [] 
batch_accuracy = [] 
for i in range(300): 
    rand_index = np.random.choice(len(x_vals), size=batch_size) 
    rand_x = x_vals[rand_index] 
    rand_y = np.transpose([y_vals[rand_index]]) 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 

    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss) 

    acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x, 
                                             y_target: rand_y, 
                                             prediction_grid:rand_x}) 
    batch_accuracy.append(acc_temp) 

x_min, x_max = x_vals[:, 0].min() - 1, x_vals[:, 0].max() + 1 
y_min, y_max = x_vals[:, 1].min() - 1, x_vals[:, 1].max() + 1 
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02), 
                     np.arange(y_min, y_max, 0.02)) 
grid_points = np.c_[xx.ravel(), yy.ravel()] 
[grid_predictions] = sess.run(prediction, feed_dict={x_data: x_vals, 
                                                   y_target: np.transpose([y_vals]), 
                                                   prediction_grid: grid_points}) 
grid_predictions = grid_predictions.reshape(xx.shape) 

plt.contourf(xx, yy, grid_predictions, cmap=plt.cm.Paired, alpha=0.8) 
plt.plot(class1_x, class1_y, 'ro', label='I. setosa') 
plt.plot(class2_x, class2_y, 'kx', label='Non-setosa') 
plt.title('Gaussian SVM Results on Iris Data') 
plt.xlabel('Petal Length') 
plt.ylabel('Sepal Width') 
plt.legend(loc='lower right') 
plt.ylim([-0.5, 3.0]) 
plt.xlim([3.5, 8.5]) 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from sklearn import datasets 
sess = tf.Session()

iris = datasets.load_iris() 
x_vals = np.array([[x[0], x[3]] for x in iris.data]) 
y_vals1 = np.array([1 if y==0 else -1 for y in iris.target]) 
y_vals2 = np.array([1 if y==1 else -1 for y in iris.target]) 
y_vals3 = np.array([1 if y==2 else -1 for y in iris.target]) 
y_vals = np.array([y_vals1, y_vals2, y_vals3]) 
class1_x = [x[0] for i,x in enumerate(x_vals) if iris.target[i]==0] 
class1_y = [x[1] for i,x in enumerate(x_vals) if iris.target[i]==0] 
class2_x = [x[0] for i,x in enumerate(x_vals) if iris.target[i]==1] 
class2_y = [x[1] for i,x in enumerate(x_vals) if iris.target[i]==1] 
class3_x = [x[0] for i,x in enumerate(x_vals) if iris.target[i]==2] 
class3_y = [x[1] for i,x in enumerate(x_vals) if iris.target[i]==2] 

batch_size = 50 

x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32) 
y_target = tf.placeholder(shape=[3, None], dtype=tf.float32) 
prediction_grid = tf.placeholder(shape=[None, 2], dtype=tf.float32) 

b = tf.Variable(tf.random_normal(shape=[3,batch_size])) 

gamma = tf.constant(-10.0) 
dist = tf.reduce_sum(tf.square(x_data), 1) 
dist = tf.reshape(dist, [-1,1]) 
sq_dists = tf.add(tf.subtract(dist, tf.multiply(2., tf.matmul(x_data, tf.transpose(x_data)))), tf.transpose(dist)) 
my_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists)))

def reshape_matmul(mat): 
    v1 = tf.expand_dims(mat, 1) 
    v2 = tf.reshape(v1, [3, batch_size, 1]) 
    return tf.batch_matmul(v2, v1)

model_output = tf.matmul(b, my_kernel) 
first_term = tf.reduce_sum(b) 
b_vec_cross = tf.matmul(tf.transpose(b), b) 
y_target_cross = reshape_matmul(y_target) 

second_term = tf.reduce_sum(tf.multiply(my_kernel, tf.multiply(b_vec_cross, y_target_cross)),[1,2]) 
loss = tf.reduce_sum(tf.negative(tf.subtract(first_term, second_term))) 

rA = tf.reshape(tf.reduce_sum(tf.square(x_data), 1),[-1,1]) 
rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1),[-1,1]) 
pred_sq_dist = tf.add(tf.subtract(rA, tf.multiply(2., tf.matmul(x_data, tf.transpose(prediction_grid)))), tf.transpose(rB)) 
pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist))) 

prediction_output = tf.matmul(tf.mul(y_target,b), pred_kernel) 
prediction = tf.arg_max(prediction_output-tf.expand_dims(tf.reduce_mean(prediction_output,1), 1), 0) 
accuracy = tf.reduce_mean(tf.cast(tf.equal(prediction, tf.argmax(y_target,0)), tf.float32)) 

my_opt = tf.train.GradientDescentOptimizer(0.01) 
train_step = my_opt.minimize(loss) 
init = tf.global_variables_initializer() 
sess.run(init)

loss_vec = [] 
batch_accuracy = [] 
for i in range(100): 
    rand_index = np.random.choice(len(x_vals), size=batch_size) 
    rand_x = x_vals[rand_index] 
    rand_y = y_vals[:,rand_index] 
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 

    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 
    loss_vec.append(temp_loss) 

    acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x, y_target: rand_y, prediction_grid:rand_x}) 
    batch_accuracy.append(acc_temp) 

    if (i+1)%25==0: 
        print('Step #' + str(i+1)) 
        print('Loss = ' + str(temp_loss)) 

Step #25 
Loss = -2.8951 
Step #50 
Loss = -27.9612 
Step #75 
Loss = -26.896 
Step #100 
Loss = -30.2325

x_min, x_max = x_vals[:, 0].min() - 1, x_vals[:, 0].max() + 1 
y_min, y_max = x_vals[:, 1].min() - 1, x_vals[:, 1].max() + 1 
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02), 
                     np.arange(y_min, y_max, 0.02)) 
grid_points = np.c_[xx.ravel(), yy.ravel()] 
grid_predictions = sess.run(prediction, feed_dict={x_data: rand_x, 
                                                   y_target: rand_y, 
                                                   prediction_grid: grid_points}) 
grid_predictions = grid_predictions.reshape(xx.shape)

plt.contourf(xx, yy, grid_predictions, cmap=plt.cm.Paired, alpha=0.8) 
plt.plot(class1_x, class1_y, 'ro', label='I. setosa') 
plt.plot(class2_x, class2_y, 'kx', label='I. versicolor') 
plt.plot(class3_x, class3_y, 'gv', label='I. virginica') 
plt.title('Gaussian SVM Results on Iris Data') 
plt.xlabel('Petal Length') 
plt.ylabel('Sepal Width') 
plt.legend(loc='lower right') 
plt.ylim([-0.5, 3.0]) 
plt.xlim([3.5, 8.5])  
plt.show() 

plt.plot(batch_accuracy, 'k-', label='Accuracy') 
plt.title('Batch Accuracy') 
plt.xlabel('Generation') 
plt.ylabel('Accuracy') 
plt.legend(loc='lower right') 
plt.show() 

plt.plot(loss_vec, 'k-') 
plt.title('Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('Loss') 
plt.show() 

import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
import requests 

sess = tf.Session() 

housing_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data' 
housing_header = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV'] 
cols_used = ['CRIM', 'INDUS', 'NOX', 'RM', 'AGE', 'DIS', 'TAX', 'PTRATIO', 'B', 'LSTAT'] 
num_features = len(cols_used) 
# Request data 
housing_file = requests.get(housing_url) 
# Parse Data 
housing_data = [[float(x) for x in y.split(' ') if len(x)>=1] for y in housing_file.text.split('n') if len(y)>=1] 

y_vals = np.transpose([np.array([y[13] for y in housing_data])]) 
x_vals = np.array([[x for i,x in enumerate(y) if housing_header[i] in cols_used] for y in housing_data]) 

x_vals = (x_vals - x_vals.min(0)) / x_vals.ptp(0) 

train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 
x_vals_train = x_vals[train_indices] 
x_vals_test = x_vals[test_indices] 
y_vals_train = y_vals[train_indices] 
y_vals_test = y_vals[test_indices] 

k = 4 
batch_size=len(x_vals_test)

x_data_train = tf.placeholder(shape=[None, num_features], dtype=tf.float32)
x_data_test = tf.placeholder(shape=[None, num_features], dtype=tf.float32)
y_target_train = tf.placeholder(shape=[None, 1], dtype=tf.float32)
y_target_test = tf.placeholder(shape=[None, 1], dtype=tf.float32) 

distance = tf.reduce_sum(tf.abs(tf.subtract(x_data_train, tf.expand_dims(x_data_test,1))), reduction_indices=2) 

top_k_xvals, top_k_indices = tf.nn.top_k(tf.negative(distance), k=k) 
x_sums = tf.expand_dims(tf.reduce_sum(top_k_xvals, 1),1) 
x_sums_repeated = tf.matmul(x_sums,tf.ones([1, k], tf.float32)) 
x_val_weights = tf.expand_dims(tf.divide(top_k_xvals,x_sums_repeated), 1) 

top_k_yvals = tf.gather(y_target_train, top_k_indices) 
prediction = tf.squeeze(tf.batch_matmul(x_val_weights,top_k_yvals), squeeze_dims=[1]) 
mse = tf.divide(tf.reduce_sum(tf.square(tf.subtract(prediction, y_target_test))), batch_size)

num_loops = int(np.ceil(len(x_vals_test)/batch_size)) 

for i in range(num_loops): 
    min_index = i*batch_size 
    max_index = min((i+1)*batch_size,len(x_vals_train)) 
    x_batch = x_vals_test[min_index:max_index] 
    y_batch = y_vals_test[min_index:max_index] 
    predictions = sess.run(prediction, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, y_target_train: y_vals_train, y_target_test: y_batch}) 
    batch_mse = sess.run(mse, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, y_target_train: y_vals_train, y_target_test: y_batch}) 

    print('Batch #' + str(i+1) + ' MSE: ' + str(np.round(batch_mse,3))) 

Batch #1 MSE: 23.153 

bins = np.linspace(5, 50, 45) 
plt.hist(predictions, bins, alpha=0.5, label='Prediction') 
plt.hist(y_batch, bins, alpha=0.5, label='Actual') 
plt.title('Histogram of Predicted and Actual Values') 
plt.xlabel('Med Home Value in $1,000s') 
plt.ylabel('Frequency') 
plt.legend(loc='upper right') 
plt.show() 

import tensorflow as tf 
sess = tf.Session() 

hypothesis = list('bear') 
truth = list('beers') 
h1 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3]], 
                     hypothesis, [1,1,1]) 
t1 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,1], [0,0,3],[0,0,4]], truth, [1,1,1]) 

print(sess.run(tf.edit_distance(h1, t1, normalize=False))) 

[[ 2.]]

hypothesis2 = list('bearbeer') 
truth2 = list('beersbeers') 
h2 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3], [0,1,0], [0,1,1], [0,1,2], [0,1,3]], hypothesis2, [1,2,4]) 
t2 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3], [0,0,4], [0,1,0], [0,1,1], [0,1,2], [0,1,3], [0,1,4]], truth2, [1,2,5]) 

print(sess.run(tf.edit_distance(h2, t2, normalize=True))) 

[[ 0.40000001  0.2      ]]

hypothesis_words = ['bear','bar','tensor','flow'] 
truth_word = ['beers''] 
num_h_words = len(hypothesis_words) 
h_indices = [[xi, 0, yi] for xi,x in enumerate(hypothesis_words) for yi,y in enumerate(x)] 
h_chars = list(''.join(hypothesis_words)) 
h3 = tf.SparseTensor(h_indices, h_chars, [num_h_words,1,1]) 
truth_word_vec = truth_word*num_h_words 
t_indices = [[xi, 0, yi] for xi,x in enumerate(truth_word_vec) for yi,y in enumerate(x)] 
t_chars = list(''.join(truth_word_vec)) 
t3 = tf.SparseTensor(t_indices, t_chars, [num_h_words,1,1]) 

print(sess.run(tf.edit_distance(h3, t3, normalize=True))) 

[[ 0.40000001]
 [ 0.60000002]
 [ 0.80000001]
 [ 1\.        ]]

def create_sparse_vec(word_list): 
    num_words = len(word_list) 
    indices = [[xi, 0, yi] for xi,x in enumerate(word_list) for yi,y in enumerate(x)] 
    chars = list(''.join(word_list)) 
    return(tf.SparseTensorValue(indices, chars, [num_words,1,1])) 

hyp_string_sparse = create_sparse_vec(hypothesis_words) 
truth_string_sparse = create_sparse_vec(truth_word*len(hypothesis_words)) 

hyp_input = tf.sparse_placeholder(dtype=tf.string) 
truth_input = tf.sparse_placeholder(dtype=tf.string) 

edit_distances = tf.edit_distance(hyp_input, truth_input, normalize=True) 

feed_dict = {hyp_input: hyp_string_sparse, 
             truth_input: truth_string_sparse} 

print(sess.run(edit_distances, feed_dict=feed_dict)) 

[[ 0.40000001]
 [ 0.60000002]
 [ 0.80000001]
 [ 1\.        ]]

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import requests
sess = tf.Session() 

housing_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data' 
housing_header = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV'] 
cols_used = ['CRIM', 'INDUS', 'NOX', 'RM', 'AGE', 'DIS', 'TAX', 'PTRATIO', 'B', 'LSTAT'] 
num_features = len(cols_used) 
housing_file = requests.get(housing_url) 
housing_data = [[float(x) for x in y.split(' ') if len(x)>=1] for y in housing_file.text.split('\n') if len(y)>=1] 
y_vals = np.transpose([np.array([y[13] for y in housing_data])]) 
x_vals = np.array([[x for i,x in enumerate(y) if housing_header[i] in cols_used] for y in housing_data]) 

x_vals = (x_vals - x_vals.min(0)) / x_vals.ptp(0)

weight_diagonal = x_vals.std(0) 
weight_matrix = tf.cast(tf.diag(weight_diagonal), dtype=tf.float32)

train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 
x_vals_train = x_vals[train_indices] 
x_vals_test = x_vals[test_indices] 
y_vals_train = y_vals[train_indices] 
y_vals_test = y_vals[test_indices] 
k = 4 
batch_size=len(x_vals_test) 

x_data_train = tf.placeholder(shape=[None, num_features], dtype=tf.float32) 
x_data_test = tf.placeholder(shape=[None, num_features], dtype=tf.float32) 
y_target_train = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
y_target_test = tf.placeholder(shape=[None, 1], dtype=tf.float32) 

subtraction_term =  tf.subtract(x_data_train, tf.expand_dims(x_data_test,1)) 
first_product = tf.batch_matmul(subtraction_term, tf.tile(tf.expand_dims(weight_matrix,0), [batch_size,1,1])) 
second_product = tf.batch_matmul(first_product, tf.transpose(subtraction_term, perm=[0,2,1])) 
distance = tf.sqrt(tf.batch_matrix_diag_part(second_product))

top_k_xvals, top_k_indices = tf.nn.top_k(tf.neg(distance), k=k) 
x_sums = tf.expand_dims(tf.reduce_sum(top_k_xvals, 1),1) 
x_sums_repeated = tf.matmul(x_sums,tf.ones([1, k], tf.float32)) 
x_val_weights = tf.expand_dims(tf.div(top_k_xvals,x_sums_repeated), 1) 
top_k_yvals = tf.gather(y_target_train, top_k_indices) 
prediction = tf.squeeze(tf.batch_matmul(x_val_weights,top_k_yvals), squeeze_dims=[1]) 

mse = tf.divide(tf.reduce_sum(tf.square(tf.subtract(prediction, y_target_test))), batch_size) 

num_loops = int(np.ceil(len(x_vals_test)/batch_size))
for i in range(num_loops):
    min_index = i*batch_size
    max_index = min((i+1)*batch_size,len(x_vals_train))
    x_batch = x_vals_test[min_index:max_index]
    y_batch = y_vals_test[min_index:max_index]
    predictions = sess.run(prediction, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, y_target_train: y_vals_train, y_target_test: y_batch})
    batch_mse = sess.run(mse, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, y_target_train: y_vals_train, y_target_test: y_batch})
    print('Batch #' + str(i+1) + ' MSE: ' + str(np.round(batch_mse,3))) 

Batch #1 MSE: 21.322

bins = np.linspace(5, 50, 45) 
plt.hist(predictions, bins, alpha=0.5, label='Prediction') 
plt.hist(y_batch, bins, alpha=0.5, label='Actual') 
plt.title('Histogram of Predicted and Actual Values') 
plt.xlabel('Med Home Value in $1,000s') 
plt.ylabel('Frequency') 
plt.legend(loc='upper right') 
plt.show() 

import random 
import string 
import numpy as np 
import tensorflow as tf 

n = 10 street_names = ['abbey', 'baker', 'canal', 'donner', 'elm']
street_types = ['rd', 'st', 'ln', 'pass', 'ave'] 
rand_zips = [random.randint(65000,65999) for i in range(5)] 
numbers = [random.randint(1, 9999) for i in range(n)] 
streets = [random.choice(street_names) for i in range(n)] 
street_suffs = [random.choice(street_types) for i in range(n)] 
zips = [random.choice(rand_zips) for i in range(n)] 
full_streets = [str(x) + ' ' + y + ' ' + z for x,y,z in zip(numbers, streets, street_suffs)] 
reference_data = [list(x) for x in zip(full_streets,zips)] 

def create_typo(s, prob=0.75):
    if random.uniform(0,1) < prob:
        rand_ind = random.choice(range(len(s)))
        s_list = list(s)
        s_list[rand_ind]=random.choice(string.ascii_lowercase)
        s = ''.join(s_list)
    return s

typo_streets = [create_typo(x) for x in streets]
typo_full_streets = [str(x) + ' ' + y + ' ' + z for x,y,z in zip(numbers, typo_streets, street_suffs)]
test_data = [list(x) for x in zip(typo_full_streets,zips)] 

sess = tf.Session() 
test_address = tf.sparse_placeholder( dtype=tf.string) 
test_zip = tf.placeholder(shape=[None, 1], dtype=tf.float32) 
ref_address = tf.sparse_placeholder(dtype=tf.string) 
ref_zip = tf.placeholder(shape=[None, n], dtype=tf.float32) 

zip_dist = tf.square(tf.subtract(ref_zip, test_zip)) 
address_dist = tf.edit_distance(test_address, ref_address, normalize=True)

zip_max = tf.gather(tf.squeeze(zip_dist), tf.argmax(zip_dist, 1)) 
zip_min = tf.gather(tf.squeeze(zip_dist), tf.argmin(zip_dist, 1)) 
zip_sim = tf.divide(tf.subtract(zip_max, zip_dist), tf.subtract(zip_max, zip_min)) 
address_sim = tf.subtract(1., address_dist) 

address_weight = 0.5 
zip_weight = 1\. - address_weight 
weighted_sim = tf.add(tf.transpose(tf.multiply(address_weight, address_sim)), tf.multiply(zip_weight, zip_sim)) 
top_match_index = tf.argmax(weighted_sim, 1) 

def sparse_from_word_vec(word_vec): 
    num_words = len(word_vec) 
    indices = [[xi, 0, yi] for xi,x in enumerate(word_vec) for yi,y in enumerate(x)] 
    chars = list(''.join(word_vec)) 
    # Now we return our sparse vector 
    return tf.SparseTensorValue(indices, chars, [num_words,1,1])

reference_addresses = [x[0] for x in reference_data] 
reference_zips = np.array([[x[1] for x in reference_data]])

sparse_ref_set = sparse_from_word_vec(reference_addresses)

for i in range(n): 
    test_address_entry = test_data[i][0] 
    test_zip_entry = [[test_data[i][1]]] 

    # Create sparse address vectors 
    test_address_repeated = [test_address_entry] * n 
    sparse_test_set = sparse_from_word_vec(test_address_repeated) 

    feeddict={test_address: sparse_test_set, 
               test_zip: test_zip_entry, 
               ref_address: sparse_ref_set, 
               ref_zip: reference_zips} 
    best_match = sess.run(top_match_index, feed_dict=feeddict)
    best_street = reference_addresses[best_match[0]]
    [best_zip] = reference_zips[0][best_match]
    [[test_zip_]] = test_zip_entry
    print('Address: ' + str(test_address_entry) + ', ' + str(test_zip_))
    print('Match : ' + str(best_street) + ', ' + str(best_zip))

Address: 8659 beker ln, 65463 
Match  : 8659 baker ln, 65463 
Address: 1048 eanal ln, 65681 
Match  : 1048 canal ln, 65681 
Address: 1756 vaker st, 65983 
Match  : 1756 baker st, 65983 
Address: 900 abbjy pass, 65983 
Match  : 900 abbey pass, 65983 
Address: 5025 canal rd, 65463 
Match  : 5025 canal rd, 65463 
Address: 6814 elh st, 65154 
Match  : 6814 elm st, 65154 
Address: 3057 cagal ave, 65463 
Match  : 3057 canal ave, 65463 
Address: 7776 iaker ln, 65681 
Match  : 7776 baker ln, 65681 
Address: 5167 caker rd, 65154

Match  : 5167 baker rd, 65154 
Address: 8765 donnor st, 65154 
Match  : 8765 donner st, 65154 

import random 
import numpy as np 
import tensorflow as tf 
import matplotlib.pyplot as plt 
from PIL import Image 
from tensorflow.examples.tutorials.mnist import input_data 

sess = tf.Session() 
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

train_size = 1000 
test_size = 102 
rand_train_indices = np.random.choice(len(mnist.train.images), train_size, replace=False) 
rand_test_indices = np.random.choice(len(mnist.test.images), test_size, replace=False) 
x_vals_train = mnist.train.images[rand_train_indices] 
x_vals_test = mnist.test.images[rand_test_indices] 
y_vals_train = mnist.train.labels[rand_train_indices] 
y_vals_test = mnist.test.labels[rand_test_indices]

k = 4 
batch_size=6 

x_data_train = tf.placeholder(shape=[None, 784], dtype=tf.float32) 
x_data_test = tf.placeholder(shape=[None, 784], dtype=tf.float32) 
y_target_train = tf.placeholder(shape=[None, 10], dtype=tf.float32) 
y_target_test = tf.placeholder(shape=[None, 10], dtype=tf.float32) 

distance = tf.reduce_sum(tf.abs(tf.subtract(x_data_train, tf.expand_dims(x_data_test,1))), reduction_indices=2) 

top_k_xvals, top_k_indices = tf.nn.top_k(tf.negative(distance), k=k) 
prediction_indices = tf.gather(y_target_train, top_k_indices) 
count_of_predictions = tf.reduce_sum(prediction_indices, reduction_indices=1) 
prediction = tf.argmax(count_of_predictions) 

num_loops = int(np.ceil(len(x_vals_test)/batch_size)) 
test_output = [] 
actual_vals = [] 
for i in range(num_loops): 
    min_index = i*batch_size 
    max_index = min((i+1)*batch_size,len(x_vals_train)) 
    x_batch = x_vals_test[min_index:max_index] 
    y_batch = y_vals_test[min_index:max_index] 
    predictions = sess.run(prediction, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, y_target_train: y_vals_train, y_target_test: y_batch}) 
    test_output.extend(predictions) 
    actual_vals.extend(np.argmax(y_batch, axis=1)) 

accuracy = sum([1./test_size for i in range(test_size) if test_output[i]==actual_vals[i]]) 
print('Accuracy on test set: ' + str(accuracy)) 
Accuracy on test set: 0.8333333333333325 

actuals = np.argmax(y_batch, axis=1) 
Nrows = 2 
Ncols = 3 
for i in range(len(actuals)): 
    plt.subplot(Nrows, Ncols, i+1) 
    plt.imshow(np.reshape(x_batch[i], [28,28]), cmap='Greys_r') 
    plt.title('Actual: ' + str(actuals[i]) + ' Pred: ' + str(predictions[i]), fontsize=10) 
    frame = plt.gca() 
    frame.axes.get_xaxis().set_visible(False) 
    frame.axes.get_yaxis().set_visible(False)