assignment2

Q1
layers.py

import numpy as np


def affine_forward(x, w, b):
    """
  Computes the forward pass for an affine (fully-connected) layer.

  The input x has shape (N, d_1, ..., d_k) and contains a minibatch of N
  examples, where each example x[i] has shape (d_1, ..., d_k). We will
  reshape each input into a vector of dimension D = d_1 * ... * d_k, and
  then transform it to an output vector of dimension M.

  Inputs:
  - x: A numpy array containing input data, of shape (N, d_1, ..., d_k)
  - w: A numpy array of weights, of shape (D, M)
  - b: A numpy array of biases, of shape (M,)

  Returns a tuple of:
  - out: output, of shape (N, M)
  - cache: (x, w, b)
  """
    out = None
    #############################################################################
    # TODO: Implement the affine forward pass. Store the result in out. You     #
    # will need to reshape the input into rows.                                 #
    #############################################################################
    D = np.product(x.shape[1:])
    N = x.shape[0]
    x1 = x.reshape((N, D))

    out = np.dot(x1, w) + b
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    cache = (x, w, b)
    return out, cache


def affine_backward(dout, cache):
    """
  Computes the backward pass for an affine layer.

  Inputs:
  - dout: Upstream derivative, of shape (N, M)
  - cache: Tuple of:
    - x: Input data, of shape (N, d_1, ... d_k)
    - w: Weights, of shape (D, M)

  Returns a tuple of:
  - dx: Gradient with respect to x, of shape (N, d1, ..., d_k)
  - dw: Gradient with respect to w, of shape (D, M)
  - db: Gradient with respect to b, of shape (M,)
  """
    x, w, b = cache
    dx, dw, db = None, None, None
    #############################################################################
    # TODO: Implement the affine backward pass.                                 #
    #############################################################################
    N = x.shape[0]
    db = np.sum(dout, axis=0)

    D = np.product(x.shape[1:])

    x1 = x.reshape((N, D))
    dw = np.dot(x1.T, dout)

    M = w.shape[1]
    D_expanded = x.shape[1:]
    w_new_shape = D_expanded + (M,)
    w1 = w.reshape(w_new_shape)
    dx = np.dot(dout, w.T)
    dx = dx.reshape(x.shape)
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    return dx, dw, db


def relu_forward(x):
    """
  Computes the forward pass for a layer of rectified linear units (ReLUs).

  Input:
  - x: Inputs, of any shape

  Returns a tuple of:
  - out: Output, of the same shape as x
  - cache: x
  """
    out = None
    #############################################################################
    # TODO: Implement the ReLU forward pass.                                    #
    #############################################################################
    out = np.copy(x)
    out[out <= 0] = 0
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    cache = x
    return out, cache


def relu_backward(dout, cache):
    """
  Computes the backward pass for a layer of rectified linear units (ReLUs).

  Input:
  - dout: Upstream derivatives, of any shape
  - cache: Input x, of same shape as dout

  Returns:
  - dx: Gradient with respect to x
  """
    dx, x = None, cache
    #############################################################################
    # TODO: Implement the ReLU backward pass.                                   #
    #############################################################################
    x[x <= 0] = 0
    x[x > 0] = 1
    dx = dout * x
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    return dx


def batchnorm_forward(x, gamma, beta, bn_param):
    """
  Forward pass for batch normalization.

  During training the sample mean and (uncorrected) sample variance are
  computed from minibatch statistics and used to normalize the incoming data.
  During training we also keep an exponentially decaying running mean of the mean
  and variance of each feature, and these averages are used to normalize data
  at test-time.

  At each timestep we update the running averages for mean and variance using
  an exponential decay based on the momentum parameter:

  running_mean = momentum * running_mean + (1 - momentum) * sample_mean
  running_var = momentum * running_var + (1 - momentum) * sample_var

  Note that the batch normalization paper suggests a different test-time
  behavior: they compute sample mean and variance for each feature using a
  large number of training images rather than using a running average. For
  this implementation we have chosen to use running averages instead since
  they do not require an additional estimation step; the torch7 implementation
  of batch normalization also uses running averages.

  Input:
  - x: Data of shape (N, D)
  - gamma: Scale parameter of shape (D,)
  - beta: Shift paremeter of shape (D,)
  - bn_param: Dictionary with the following keys:
    - mode: 'train' or 'test'; required
    - eps: Constant for numeric stability
    - momentum: Constant for running mean / variance.
    - running_mean: Array of shape (D,) giving running mean of features
    - running_var Array of shape (D,) giving running variance of features

  Returns a tuple of:
  - out: of shape (N, D)
  - cache: A tuple of values needed in the backward pass
  """
    mode = bn_param['mode']
    eps = bn_param.get('eps', 1e-5)
    momentum = bn_param.get('momentum', 0.9)

    N, D = x.shape
    running_mean = bn_param.get('running_mean', np.zeros(D, dtype=x.dtype))
    running_var = bn_param.get('running_var', np.zeros(D, dtype=x.dtype))

    out, cache = None, None
    if mode == 'train':
        #############################################################################
        # TODO: Implement the training-time forward pass for batch normalization.   #
        # Use minibatch statistics to compute the mean and variance, use these      #
        # statistics to normalize the incoming data, and scale and shift the        #
        # normalized data using gamma and beta.                                     #
        #                                                                           #
        # You should store the output in the variable out. Any intermediates that   #
        # you need for the backward pass should be stored in the cache variable.    #
        #                                                                           #
        # You should also use your computed sample mean and variance together with  #
        # the momentum variable to update the running mean and running variance,    #
        # storing your result in the running_mean and running_var variables.        #
        #############################################################################
        u = np.mean(x, axis=0)
        numerator = x - u
        v = np.var(x, axis=0)
        v_plus_eps = v + eps
        denominator = np.sqrt(v_plus_eps)
        inv_denominator = 1. / denominator
        xhat = numerator * inv_denominator
        gamma_mul_xhat = gamma * xhat
        y = gamma_mul_xhat + beta
        out = y
        cache = x, u, numerator, v, v_plus_eps, denominator, inv_denominator, xhat, gamma_mul_xhat, y, gamma, beta

        running_mean = momentum * running_mean + (1 - momentum) * u
        running_var = momentum * running_var + (1 - momentum) * v
        bn_param['running_mean'] = running_mean
        bn_param['running_var'] = running_var
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################
    elif mode == 'test':
        #############################################################################
        # TODO: Implement the test-time forward pass for batch normalization. Use   #
        # the running mean and variance to normalize the incoming data, then scale  #
        # and shift the normalized data using gamma and beta. Store the result in   #
        # the out variable.                                                         #
        #############################################################################
        u = bn_param['running_mean']
        numerator = x - u
        v = bn_param['running_var']
        v_plus_eps = v + eps
        denominator = np.sqrt(v_plus_eps)
        inv_denominator = 1. / denominator
        xhat = numerator * inv_denominator
        gamma_mul_xhat = gamma * xhat
        y = gamma_mul_xhat + beta
        out = y
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################
    else:
        raise ValueError('Invalid forward batchnorm mode "%s"' % mode)

    # Store the updated running means back into bn_param
    bn_param['running_mean'] = running_mean
    bn_param['running_var'] = running_var

    return out, cache


def batchnorm_backward(dout, cache):
    """
  Backward pass for batch normalization.

  For this implementation, you should write out a computation graph for
  batch normalization on paper and propagate gradients backward through
  intermediate nodes.

  Inputs:
  - dout: Upstream derivatives, of shape (N, D)
  - cache: Variable of intermediates from batchnorm_forward.

  Returns a tuple of:
  - dx: Gradient with respect to inputs x, of shape (N, D)
  - dgamma: Gradient with respect to scale parameter gamma, of shape (D,)
  - dbeta: Gradient with respect to shift parameter beta, of shape (D,)
  """
    dx, dgamma, dbeta = None, None, None
    x, u, numerator, v, v_plus_eps, denominator, inv_denominator, xhat, gamma_mul_xhat, y, gamma, beta = cache

    #############################################################################
    # TODO: Implement the backward pass for batch normalization. Store the      #
    # results in the dx, dgamma, and dbeta variables.                           #
    #############################################################################
    dx = np.zeros(dout.shape)
    N = dout.shape[0]
    dbeta = np.sum(dout, axis=0)
    dgamma = np.sum(dout * xhat, axis=0)

    dxhat = dout * gamma

    dinv_den = np.sum(dxhat * numerator, axis=0)
    dsqrt_v_plus_eps = dinv_den * (-1) * (denominator ** (-2))
    dv_plus_eps = dsqrt_v_plus_eps * 0.5 * (v_plus_eps ** (-0.5))
    dv = 2. / N * (x - u) * dv_plus_eps
    dx += dv
    du1 = -np.sum(dv, axis=0)
    dux = np.full(dout.shape, 1. / N)
    du1_x = du1 * dux
    dx += du1_x

    dnumerator = dxhat * inv_denominator
    dx += dnumerator
    du2 = -np.sum(dnumerator, axis=0)
    du2_x = du2 * dux
    dx += du2_x

    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################

    return dx, dgamma, dbeta


def dropout_forward(x, dropout_param):
    """
  Performs the forward pass for (inverted) dropout.

  Inputs:
  - x: Input data, of any shape
  - dropout_param: A dictionary with the following keys:
    - p: Dropout parameter. We drop each neuron output with probability p.
    - mode: 'test' or 'train'. If the mode is train, then perform dropout;
      if the mode is test, then just return the input.
    - seed: Seed for the random number generator. Passing seed makes this
      function deterministic, which is needed for gradient checking but not in
      real networks.

  Outputs:
  - out: Array of the same shape as x.
  - cache: A tuple (dropout_param, mask). In training mode, mask is the dropout
    mask that was used to multiply the input; in test mode, mask is None.
  """
    p, mode = dropout_param['p'], dropout_param['mode']
    if 'seed' in dropout_param:
        np.random.seed(dropout_param['seed'])

    mask = None
    out = None

    mask = np.random.random(x.shape)
    mask[mask < p] = 0
    mask[mask >= p] = 1

    # mask=(np.random.random(x.shape) > p) / (1 - p)


    if mode == 'train':
        ###########################################################################
        # TODO: Implement the training phase forward pass for inverted dropout.   #
        # Store the dropout mask in the mask variable.                            #
        ###########################################################################
        out = x * mask
        ###########################################################################
        #                            END OF YOUR CODE                             #
        ###########################################################################
    elif mode == 'test':
        ###########################################################################
        # TODO: Implement the test phase forward pass for inverted dropout.       #
        ###########################################################################
        out = x * (1 - p)
        # out = x
        ###########################################################################
        #                            END OF YOUR CODE                             #
        ###########################################################################

    cache = (dropout_param, mask)
    out = out.astype(x.dtype, copy=False)

    return out, cache


def dropout_backward(dout, cache):
    """
  Perform the backward pass for (inverted) dropout.

  Inputs:
  - dout: Upstream derivatives, of any shape
  - cache: (dropout_param, mask) from dropout_forward.
  """
    dropout_param, mask = cache
    mode = dropout_param['mode']
    dx = None
    if mode == 'train':
        ###########################################################################
        # TODO: Implement the training phase backward pass for inverted dropout.  #
        ###########################################################################
        dx = np.ones(mask.shape) * mask
        dx *= dout
        ###########################################################################
        #                            END OF YOUR CODE                             #
        ###########################################################################
    elif mode == 'test':
        dx = dout
    return dx


def conv_forward_naive(x, w, b, conv_param):
    """
  A naive implementation of the forward pass for a convolutional layer.

  The input consists of N data points, each with C channels, height H and width
  W. We convolve each input with F different filters, where each filter spans
  all C channels and has height HH and width HH.

  Input:
  - x: Input data of shape (N, C, H, W)
  - w: Filter weights of shape (F, C, HH, WW)
  - b: Biases, of shape (F,)
  - conv_param: A dictionary with the following keys:
    - 'stride': The number of pixels between adjacent receptive fields in the
      horizontal and vertical directions.
    - 'pad': The number of pixels that will be used to zero-pad the input.

  Returns a tuple of:
  - out: Output data, of shape (N, F, H', W') where H' and W' are given by
    H' = 1 + (H + 2 * pad - HH) / stride
    W' = 1 + (W + 2 * pad - WW) / stride
  - cache: (x, w, b, conv_param)
  """
    out = None
    #############################################################################
    # TODO: Implement the convolutional forward pass.                           #
    # Hint: you can use the function np.pad for padding.                        #
    #############################################################################
    pad = conv_param['pad']
    stride = conv_param['stride']
    N, C, H, W = x.shape
    F, C, HH, WW = w.shape
    x = np.pad(x, ((0, 0), (0, 0), (pad, pad), (pad, pad)), 'constant', constant_values=0)
    WOUT = (W - WW + 2 * pad) / stride + 1
    HOUT = (H - HH + 2 * pad) / stride + 1
    t = np.zeros((N, WW * HH * C, WOUT * HOUT))
    for i in range(0, N):
        x_i = x[i, :, :, :]
        # for each image
        l = 0
        for j in range(0, HOUT):
            for k in range(0, WOUT):
                # strech out filter into a col vector
                slide_filter = x_i[:, j * stride:j * stride + HH, k * stride:k * stride + WW]
                slide_filter_col = slide_filter.reshape((WW * HH * C))
                # there are WOUT * HOUT such col vector
                t[i, :, l] = slide_filter_col
                l += 1
    w_reshape = w.reshape((F, C * HH * WW))
    out = np.zeros((N, F, WOUT * HOUT))
    for i in range(0, N):
        out[i] = w_reshape.dot(t[i]) + b.reshape((F, 1))
    out = out.reshape((N, F, HOUT, WOUT))
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    cache = (x, w, b, conv_param)
    return out, cache


def conv_backward_naive(dout, cache):
    """
  A naive implementation of the backward pass for a convolutional layer.

  Inputs:
  - dout: Upstream derivatives.
  - cache: A tuple of (x, w, b, conv_param) as in conv_forward_naive

  Returns a tuple of:
  - dx: Gradient with respect to x
  - dw: Gradient with respect to w
  - db: Gradient with respect to b
  """
    dx, dw, db = None, None, None
    #############################################################################
    # TODO: Implement the convolutional backward pass.                          #
    #############################################################################
    # db
    db1 = np.sum(dout, axis=0)
    db2 = np.sum(db1, axis=1)
    db = np.sum(db2, axis=1)

    # dx
    # x is after padding
    x, w, b, conv_param = cache
    N = x.shape[0]
    C = w.shape[1]
    F = dout.shape[1]
    HH = w.shape[2]
    WW = w.shape[3]
    HOUT = dout.shape[2]
    WOUT = dout.shape[3]
    stride = conv_param['stride']
    # get t from x
    t = np.zeros((N, WW * HH * C, WOUT * HOUT))
    for i in range(0, N):
        x_i = x[i, :, :, :]
        # for each image
        l = 0
        for j in range(0, HOUT):
            for k in range(0, WOUT):
                # strech out filter into a col vector
                slide_filter = x_i[:, j * stride:j * stride + HH, k * stride:k * stride + WW]
                slide_filter_col = slide_filter.reshape((WW * HH * C))
                # there are WOUT * HOUT such col vector
                t[i, :, l] = slide_filter_col
                l += 1
    # dout,(N=4,F=2,HOUT=5,WOUT=5)
    # dw,(F=2, C=3, HH=3, WW=3)
    # t,(N=4,(WW * HH * C),(HOUT * WOUT))
    dout = dout.reshape(N, F, (HOUT * WOUT))
    t = np.transpose(t, axes=(0, 2, 1))
    dw = np.zeros(w.shape)
    for i in range(0, N):
        t_i = t[i, :, :]
        dout_i = dout[i, :, :]
        # dout_i:(F,(HOUT * WOUT)) after reshaping, t_i:((HOUT * WOUT),(WW * HH * C)) after transposing
        dw_i = dout_i.dot(t_i)
        dw_i = dw_i.reshape((F, C, HH, WW))
        dw += dw_i

    # dx
    # dx:(N,C,HOUT,WOUT),dout:(N,F,(HOUT * WOUT)),w:(F=2,(C*HH*WW))
    w = w.reshape((F, (C * HH * WW)))
    # dout:(N,(HOUT * WOUT),F)
    dout = np.transpose(dout, axes=(0, 2, 1))
    # dt:(N, (HOUT * WOUT), (C*HH*WW))
    dt = dout.dot(w)
    # dt:(N, (C*HH*WW),(HOUT * WOUT))
    dt = np.transpose(dt, axes=(0, 2, 1))
    dx = np.zeros(x.shape)
    # recover dx from dt
    for i in range(0, N):
        x_i = dx[i, :, :, :]
        l = 0
        for j in range(0, HOUT):
            for k in range(0, WOUT):
                slide_filter_col = dt[i, :, l]
                slide_filter = slide_filter_col.reshape((C, HH, WW))
                # += not =, overlapping is needed
                x_i[:, j * stride:j * stride + HH, k * stride:k * stride + WW] += slide_filter
                l += 1
    # delete padding
    dx = dx[:, :, 1:-1, 1:-1]

    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    return dx, dw, db


def max_pool_forward_naive(x, pool_param):
    """
  A naive implementation of the forward pass for a max pooling layer.

  Inputs:
  - x: Input data, of shape (N, C, H, W)
  - pool_param: dictionary with the following keys:
    - 'pool_height': The height of each pooling region
    - 'pool_width': The width of each pooling region
    - 'stride': The distance between adjacent pooling regions

  Returns a tuple of:
  - out: Output data
  - cache: (x, pool_param)
  """
    out = None
    #############################################################################
    # TODO: Implement the max pooling forward pass                    #
    #############################################################################
    pool_height = pool_param['pool_height']
    pool_width = pool_param['pool_width']
    stride = pool_param['stride']
    N, C, H, W = x.shape

    HOUT = (H - pool_height) / stride + 1
    WOUT = (W - pool_width) / stride + 1
    out = np.zeros((N, C, HOUT, WOUT))
    for i in range(0, N):
        x_i = x[i, :, :, :]
        for j in range(0, HOUT):
            for k in range(0, WOUT):
                slide_filter = x_i[:, j * stride:j * stride + pool_height, k * stride:k * stride + pool_width]
                max_entry = np.max(slide_filter, axis=(1, 2))
                out[i, :, j, k] = max_entry
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    cache = (x, pool_param)
    return out, cache


def max_pool_backward_naive(dout, cache):
    """
  A naive implementation of the backward pass for a max pooling layer.

  Inputs:
  - dout: Upstream derivatives
  - cache: A tuple of (x, pool_param) as in the forward pass.

  Returns:
  - dx: Gradient with respect to x
  """
    dx = None
    #############################################################################
    # TODO: Implement the max pooling backward pass                             #
    #############################################################################
    x, pool_param = cache
    pool_height = pool_param['pool_height']
    pool_width = pool_param['pool_width']
    stride = pool_param['stride']
    N, C, H, W = x.shape
    HOUT = (H - pool_height) / stride + 1
    WOUT = (W - pool_width) / stride + 1
    dx = np.zeros(x.shape)
    for i in range(0, N):
        x_i = x[i, :, :, :]
        for j in range(0, HOUT):
            for k in range(0, WOUT):
                slide_filter = x_i[:, j * stride:j * stride + pool_height, k * stride:k * stride + pool_width]
                # now 2 dimensional
                slide_filter_streched = slide_filter.reshape((C, pool_height * pool_width))
                indexes = np.argmax(slide_filter_streched, axis=1)
                # compute max entry's relative h,w in a filter
                for t in range(0, len(indexes)):
                    index = indexes[t]
                    h = index / pool_width
                    w = index % pool_width
                    # recover the absolute position in dx
                    # the gradient is 1, so the final value is just upstream derivates by chain rule
                    dx[i][t][j * stride + h][k * stride + w] = dout[i][t][j][k]
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    return dx


def spatial_batchnorm_forward(x, gamma, beta, bn_param):
    """
  Computes the forward pass for spatial batch normalization.

  Inputs:
  - x: Input data of shape (N, C, H, W)
  - gamma: Scale parameter, of shape (C,)
  - beta: Shift parameter, of shape (C,)
  - bn_param: Dictionary with the following keys:
    - mode: 'train' or 'test'; required
    - eps: Constant for numeric stability
    - momentum: Constant for running mean / variance. momentum=0 means that
      old information is discarded completely at every time step, while
      momentum=1 means that new information is never incorporated. The
      default of momentum=0.9 should work well in most situations.
    - running_mean: Array of shape (D,) giving running mean of features
    - running_var Array of shape (D,) giving running variance of features

  Returns a tuple of:
  - out: Output data, of shape (N, C, H, W)
  - cache: Values needed for the backward pass
  """
    out, cache = None, None

    #############################################################################
    # TODO: Implement the forward pass for spatial batch normalization.         #
    #                                                                           #
    # HINT: You can implement spatial batch normalization using the vanilla     #
    # version of batch normalization defined above. Your implementation should  #
    # be very short; ours is less than five lines.                              #
    #############################################################################
    N, C, H, W = x.shape
    # x:(N,C,H,W)->(C,N,H,W)
    x_NC_transposed = np.transpose(x, axes=(1,0,2,3))
    # stretch (H,W) to (H*W)
    x_NC_transposed_reshaped = x_NC_transposed.reshape(C,N,H*W)
    # result holder
    out = np.zeros((C,N,H*W))
    cache=[]
    # call previous batchnorm forward per depth
    for i in range(0, C):
      out[i],cachei = batchnorm_forward(x_NC_transposed_reshaped[i], gamma[i], beta[i], bn_param)
      cache.append(cachei)
    # recover result in demanded form
    out = np.transpose(out.reshape(C,N,H,W),axes=(1,0,2,3))
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################

    return out, cache


def spatial_batchnorm_backward(dout, cache):
    """
  Computes the backward pass for spatial batch normalization.

  Inputs:
  - dout: Upstream derivatives, of shape (N, C, H, W)
  - cache: Values from the forward pass

  Returns a tuple of:
  - dx: Gradient with respect to inputs, of shape (N, C, H, W)
  - dgamma: Gradient with respect to scale parameter, of shape (C,)
  - dbeta: Gradient with respect to shift parameter, of shape (C,)
  """
    dx, dgamma, dbeta = None, None, None

    #############################################################################
    # TODO: Implement the backward pass for spatial batch normalization.        #
    #                                                                           #
    # HINT: You can implement spatial batch normalization using the vanilla     #
    # version of batch normalization defined above. Your implementation should  #
    # be very short; ours is less than five lines.                              #
    #############################################################################
    N, C, H, W = dout.shape
    # dout:(N,C,H,W)->(C,N,H,W)
    dout_NC_transposed = np.transpose(dout,axes=(1,0,2,3))
    # stretch (H,W) to (H*W)
    dout_transposed_reshaped = dout_NC_transposed.reshape((C,N,H*W))
    # result holders
    dx = np.zeros((C,N,H*W))
    dgamma = np.zeros((C,))
    dbeta = np.zeros((C,))
    # call previous batchnorm backward per depth
    for i in range(0,C):
      dxi, dgammai, dbetai = batchnorm_backward(dout_transposed_reshaped[i],cache[i])
      dx[i] = dxi
      # note dgammai and dbetai are (H*W), but we want each depth slice has only one gamma and beta
      # so we just sum over the 2d matrix to obtain a single value
      dgamma[i] = np.sum(dgammai)
      dbeta[i] = np.sum(dbetai)
    # recover result in demanded form
    dx = np.transpose(dx.reshape((C,N,H,W)),axes=(1,0,2,3))
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################

    return dx, dgamma, dbeta


def svm_loss(x, y):
    """
  Computes the loss and gradient using for multiclass SVM classification.

  Inputs:
  - x: Input data, of shape (N, C) where x[i, j] is the score for the jth class
    for the ith input.
  - y: Vector of labels, of shape (N,) where y[i] is the label for x[i] and
    0 <= y[i] < C

  Returns a tuple of:
  - loss: Scalar giving the loss
  - dx: Gradient of the loss with respect to x
  """
    N = x.shape[0]
    correct_class_scores = x[np.arange(N), y]
    margins = np.maximum(0, x - correct_class_scores[:, np.newaxis] + 1.0)
    margins[np.arange(N), y] = 0
    loss = np.sum(margins) / N
    num_pos = np.sum(margins > 0, axis=1)
    dx = np.zeros_like(x)
    dx[margins > 0] = 1
    dx[np.arange(N), y] -= num_pos
    dx /= N
    return loss, dx


def softmax_loss(x, y):
    """
  Computes the loss and gradient for softmax classification.

  Inputs:
  - x: Input data, of shape (N, C) where x[i, j] is the score for the jth class
    for the ith input.
  - y: Vector of labels, of shape (N,) where y[i] is the label for x[i] and
    0 <= y[i] < C

  Returns a tuple of:
  - loss: Scalar giving the loss
  - dx: Gradient of the loss with respect to x
  """
    probs = np.exp(x - np.max(x, axis=1, keepdims=True))
    probs /= np.sum(probs, axis=1, keepdims=True)
    N = x.shape[0]
    loss = -np.sum(np.log(probs[np.arange(N), y])) / N
    dx = probs.copy()
    dx[np.arange(N), y] -= 1
    dx /= N
    return loss, dx

fc_net.py

import numpy as np

from cs231n.layers import *
from cs231n.layer_utils import *


class TwoLayerNet(object):
  """
  A two-layer fully-connected neural network with ReLU nonlinearity and
  softmax loss that uses a modular layer design. We assume an input dimension
  of D, a hidden dimension of H, and perform classification over C classes.
  
  The architecure should be affine - relu - affine - softmax.

  Note that this class does not implement gradient descent; instead, it
  will interact with a separate Solver object that is responsible for running
  optimization.

  The learnable parameters of the model are stored in the dictionary
  self.params that maps parameter names to numpy arrays.
  """
  
  def __init__(self, input_dim=3*32*32, hidden_dim=100, num_classes=10,
               weight_scale=1e-3, reg=0.0):
    """
    Initialize a new network.

    Inputs:
    - input_dim: An integer giving the size of the input
    - hidden_dim: An integer giving the size of the hidden layer
    - num_classes: An integer giving the number of classes to classify
    - dropout: Scalar between 0 and 1 giving dropout strength.
    - weight_scale: Scalar giving the standard deviation for random
      initialization of the weights.
    - reg: Scalar giving L2 regularization strength.
    """
    self.params = {}
    self.reg = reg
    
    ############################################################################
    # TODO: Initialize the weights and biases of the two-layer net. Weights    #
    # should be initialized from a Gaussian with standard deviation equal to   #
    # weight_scale, and biases should be initialized to zero. All weights and  #
    # biases should be stored in the dictionary self.params, with first layer  #
    # weights and biases using the keys 'W1' and 'b1' and second layer weights #
    # and biases using the keys 'W2' and 'b2'.                                 #
    ############################################################################
    W1 = np.random.randn(input_dim,hidden_dim) * weight_scale
    b1 = np.zeros(hidden_dim)
    W2 = np.random.randn(hidden_dim, num_classes) * weight_scale
    b2 = np.zeros(num_classes)
    self.params['W1'] = W1 
    self.params['b1'] =  b1
    self.params['W2'] = W2
    self.params['b2'] = b2
    ############################################################################
    #                             END OF YOUR CODE                             #
    ############################################################################


  def loss(self, X, y=None):
    """
    Compute loss and gradient for a minibatch of data.

    Inputs:
    - X: Array of input data of shape (N, d_1, ..., d_k)
    - y: Array of labels, of shape (N,). y[i] gives the label for X[i].

    Returns:
    If y is None, then run a test-time forward pass of the model and return:
    - scores: Array of shape (N, C) giving classification scores, where
      scores[i, c] is the classification score for X[i] and class c.

    If y is not None, then run a training-time forward and backward pass and
    return a tuple of:
    - loss: Scalar value giving the loss
    - grads: Dictionary with the same keys as self.params, mapping parameter
      names to gradients of the loss with respect to those parameters.
    """
    
    W1 = self.params['W1'] 
    b1 = self.params['b1']
    W2 = self.params['W2']
    b2 =self.params['b2']
    
    scores = None
    ############################################################################
    # TODO: Implement the forward pass for the two-layer net, computing the    #
    # class scores for X and storing them in the scores variable.              #
    ############################################################################
    dout1, cache_1 = affine_relu_forward(X, W1, b1)
    dout2, cache_2 = affine_forward(dout1, W2, b2)
    scores = dout2
    ############################################################################
    #                             END OF YOUR CODE                             #
    ############################################################################

    # If y is None then we are in test mode so just return scores
    if y is None:
      return scores
    
    loss, grads = 0, {}
    ############################################################################
    # TODO: Implement the backward pass for the two-layer net. Store the loss  #
    # in the loss variable and gradients in the grads dictionary. Compute data #
    # loss using softmax, and make sure that grads[k] holds the gradients for  #
    # self.params[k]. Don't forget to add L2 regularization!                   #
    #                                                                          #
    # NOTE: To ensure that your implementation matches ours and you pass the   #
    # automated tests, make sure that your L2 regularization includes a factor #
    # of 0.5 to simplify the expression for the gradient.                      #
    ############################################################################

    loss, dx = softmax_loss(dout2, y)
    loss += 0.5 * self.reg * (np.sum(W1*W1) + np.sum(W2*W2))
    
    dx2, dw2, db2 = affine_backward(dx, cache_2)
    dx1, dw1, db1 = affine_relu_backward(dx2, cache_1)
    grads['W1'] = dw1 + self.reg * W1
    grads['b1'] = db1
    grads['W2'] = dw2 + self.reg * W2
    grads['b2'] = db2
    ############################################################################
    #                             END OF YOUR CODE                             #
    ############################################################################

    return loss, grads


class FullyConnectedNet(object):
  """
  A fully-connected neural network with an arbitrary number of hidden layers,
  ReLU nonlinearities, and a softmax loss function. This will also implement
  dropout and batch normalization as options. For a network with L layers,
  the architecture will be
  
  {affine - [batch norm] - relu - [dropout]} x (L - 1) - affine - softmax
  
  where batch normalization and dropout are optional, and the {...} block is
  repeated L - 1 times.
  
  Similar to the TwoLayerNet above, learnable parameters are stored in the
  self.params dictionary and will be learned using the Solver class.
  """

  def __init__(self, hidden_dims, input_dim=3*32*32, num_classes=10,
               dropout=0, use_batchnorm=False, reg=0.0,
               weight_scale=1e-2, dtype=np.float32, seed=None):
    """
    Initialize a new FullyConnectedNet.
    
    Inputs:
    - hidden_dims: A list of integers giving the size of each hidden layer.
    - input_dim: An integer giving the size of the input.
    - num_classes: An integer giving the number of classes to classify.
    - dropout: Scalar between 0 and 1 giving dropout strength. If dropout=0 then
      the network should not use dropout at all.
    - use_batchnorm: Whether or not the network should use batch normalization.
    - reg: Scalar giving L2 regularization strength.
    - weight_scale: Scalar giving the standard deviation for random
      initialization of the weights.
    - dtype: A numpy datatype object; all computations will be performed using
      this datatype. float32 is faster but less accurate, so you should use
      float64 for numeric gradient checking.
    - seed: If not None, then pass this random seed to the dropout layers. This
      will make the dropout layers deteriminstic so we can gradient check the
      model.
    """
    self.use_batchnorm = use_batchnorm
    self.use_dropout = dropout > 0
    self.reg = reg
    self.num_layers = 1 + len(hidden_dims)
    self.dtype = dtype
    self.params = {}

    ############################################################################
    # TODO: Initialize the parameters of the network, storing all values in    #
    # the self.params dictionary. Store weights and biases for the first layer #
    # in W1 and b1; for the second layer use W2 and b2, etc. Weights should be #
    # initialized from a normal distribution with standard deviation equal to  #
    # weight_scale and biases should be initialized to zero.                   #
    #                                                                          #
    # When using batch normalization, store scale and shift parameters for the #
    # first layer in gamma1 and beta1; for the second layer use gamma2 and     #
    # beta2, etc. Scale parameters should be initialized to one and shift      #
    # parameters should be initialized to zero.                                #
    ############################################################################
    fan_in, fan_out = input_dim, hidden_dims[0]
    for i in range(0, len(hidden_dims)):
        fan_out = hidden_dims[i]
        self.params['W'+str(i+1)] = np.random.randn(fan_in, fan_out) * weight_scale
        self.params['b'+str(i+1)] = np.zeros(fan_out)
        fan_in = fan_out
        if self.use_batchnorm == True:
            self.params["gamma"+str(i+1)] = np.ones((1,fan_out))
            self.params["beta"+str(i+1)] = np.zeros((1,fan_out))
    self.params['W'+ str(self.num_layers)] = np.random.randn(hidden_dims[-1], num_classes) * weight_scale
    self.params['b'+ str(self.num_layers)] = np.zeros(num_classes)
    ############################################################################
    #                             END OF YOUR CODE                             #
    ############################################################################

    # When using dropout we need to pass a dropout_param dictionary to each
    # dropout layer so that the layer knows the dropout probability and the mode
    # (train / test). You can pass the same dropout_param to each dropout layer.
    self.dropout_param = {}
    if self.use_dropout:
      self.dropout_param = {'mode': 'train', 'p': dropout}
      if seed is not None:
        self.dropout_param['seed'] = seed
    
    # With batch normalization we need to keep track of running means and
    # variances, so we need to pass a special bn_param object to each batch
    # normalization layer. You should pass self.bn_params[0] to the forward pass
    # of the first batch normalization layer, self.bn_params[1] to the forward
    # pass of the second batch normalization layer, etc.
    self.bn_params = []
    if self.use_batchnorm:
      self.bn_params = [{'mode': 'train'} for i in xrange(self.num_layers - 1)]
    
    # Cast all parameters to the correct datatype
    for k, v in self.params.iteritems():
      self.params[k] = v.astype(dtype)

  def loss(self, X, y=None):
    """
    Compute loss and gradient for the fully-connected net.

    Input / output: Same as TwoLayerNet above.
    """
    X = X.astype(self.dtype)
    mode = 'test' if y is None else 'train'

    # Set train/test mode for batchnorm params and dropout param since they
    # behave differently during training and testing.
    if self.dropout_param is not None:
      self.dropout_param['mode'] = mode   
    if self.use_batchnorm:
      for bn_param in self.bn_params:
        bn_param[mode] = mode

    scores = None
    ############################################################################
    # TODO: Implement the forward pass for the fully-connected net, computing  #
    # the class scores for X and storing them in the scores variable.          #
    #                                                                          #
    # When using dropout, you'll need to pass self.dropout_param to each       #
    # dropout forward pass.                                                    #
    #                                                                          #
    # When using batch normalization, you'll need to pass self.bn_params[0] to #
    # the forward pass for the first batch normalization layer, pass           #
    # self.bn_params[1] to the forward pass for the second batch normalization #
    # layer, etc.                                                              #
    ############################################################################
    if self.use_batchnorm == True:
        len_hidden_dims = self.num_layers - 1
        caches = []
        out = X
        #print self.params.keys()
        for i in range(0, len_hidden_dims):
            w = self.params["W"+str(i+1)]
            b = self.params["b"+str(i+1)]
            # affine
            out, cache1 = affine_forward(out, w, b)
            # batch normalization, need not worry mode as it is in bn_param[i]
            gamma = self.params["gamma"+str(i+1)]
            beta = self.params["beta"+str(i+1)]
            out, cache2 = batchnorm_forward(out, gamma, beta, self.bn_params[i])
            # relu
            out, cache3 = relu_forward(out)
            if self.use_dropout == True:
                # dropout
                out, cache4 = dropout_forward(out, self.dropout_param)
            # merge cache:
            # cache[0] affine_forward
            # cache[1] batch normalization
            # cache[2] relu
            cache = []
            cache.append(cache1)
            cache.append(cache2)
            cache.append(cache3)
            if self.use_dropout == True:
                # dropout
                cache.append(cache4)
            caches.append(cache)

        w_last_layer = self.params['W'+ str(self.num_layers)]
        b_last_layer = self.params['b'+ str(self.num_layers)]
        out,cache = affine_forward(out, w_last_layer, b_last_layer)
        scores = out
        caches.append(cache)
    if self.use_batchnorm == False:
        len_hidden_dims = self.num_layers - 1
        caches = []
        out = X
        #print self.params.keys()
        for i in range(0, len_hidden_dims):
            w = self.params["W"+str(i+1)]
            b = self.params["b"+str(i+1)]
            cache = []
            # affine_relu
            out, cache1 = affine_relu_forward(out, w, b)
            if self.use_dropout == True:
                # dropout
                out, cache2 = dropout_forward(out, self.dropout_param)
            # merge cache
            cache.append(cache1)
            if self.use_dropout == True:
                cache.append(cache2)
            caches.append(cache)

        w_last_layer = self.params['W'+ str(self.num_layers)]
        b_last_layer = self.params['b'+ str(self.num_layers)]
        out,cache = affine_forward(out, w_last_layer, b_last_layer)
        scores = out
        caches.append(cache)
    ############################################################################
    #                             END OF YOUR CODE                             #
    ############################################################################

    # If test mode return early
    if mode == 'test':
      return scores

    loss, grads = 0.0, {}
    ############################################################################
    # TODO: Implement the backward pass for the fully-connected net. Store the #
    # loss in the loss variable and gradients in the grads dictionary. Compute #
    # data loss using softmax, and make sure that grads[k] holds the gradients #
    # for self.params[k]. Don't forget to add L2 regularization!               #
    #                                                                          #
    # When using batch normalization, you don't need to regularize the scale   #
    # and shift parameters.                                                    #
    #                                                                          #
    # NOTE: To ensure that your implementation matches ours and you pass the   #
    # automated tests, make sure that your L2 regularization includes a factor #
    # of 0.5 to simplify the expression for the gradient.                      #
    ############################################################################
    #print "out.shape",out.shape,"y.shape",y.shape
    if self.use_batchnorm == True:
        loss, grad = softmax_loss(out, y)
        # loss regulazation
        for i in range(1, self.num_layers+1):
            loss += 0.5 * self.reg * np.sum(self.params["W"+str(i)]*self.params["W"+str(i)]) 
        dout = grad
        caches.reverse()
        dout, dw, db = affine_backward(dout, caches[0])
        # weight regulazation
        grads['W'+ str(self.num_layers)] = dw + self.reg * self.params['W'+ str(self.num_layers)]
        grads['b'+ str(self.num_layers)] = db
        for i in range(1, len(caches)):
            if self.use_dropout == True:
                dout = dropout_backward(dout, caches[i][3])
            dout = relu_backward(dout, caches[i][2])
            dout, dgamma, dbeta = batchnorm_backward(dout, caches[i][1])
            dout, dw, db = affine_backward(dout, caches[i][0])
            grads['W'+ str(self.num_layers-i)] = dw + self.reg * self.params['W'+ str(self.num_layers-i)]
            grads['b'+ str(self.num_layers-i)] = db
            grads['gamma'+ str(self.num_layers-i)] = dgamma
            grads['beta'+ str(self.num_layers-i)] = dbeta
    if self.use_batchnorm == False:  
        loss, grad = softmax_loss(out, y)
        # loss regulazation
        for i in range(1, self.num_layers+1):
            loss += 0.5 * self.reg * np.sum(self.params["W"+str(i)]*self.params["W"+str(i)]) 
        dx = grad
        caches.reverse()
        dx, dw, db = affine_backward(dx, caches[0])
        # weight regulazation
        grads['W'+ str(self.num_layers)] = dw + self.reg * self.params['W'+ str(self.num_layers)]
        grads['b'+ str(self.num_layers)] = db
        for i in range(1, len(caches)):
            if self.use_dropout == True:
                dx = dropout_backward(dx, caches[i][1])
            dx, dw, db = affine_relu_backward(dx, caches[i][0])
            grads['W'+ str(self.num_layers-i)] = dw + self.reg * self.params['W'+ str(self.num_layers-i)]
            grads['b'+ str(self.num_layers-i)] = db
    ############################################################################
    #                             END OF YOUR CODE                             #
    ############################################################################

    return loss, grads

solver.py

import numpy as np

from cs231n import optim


class Solver(object):
  """
  A Solver encapsulates all the logic necessary for training classification
  models. The Solver performs stochastic gradient descent using different
  update rules defined in optim.py.

  The solver accepts both training and validataion data and labels so it can
  periodically check classification accuracy on both training and validation
  data to watch out for overfitting.

  To train a model, you will first construct a Solver instance, passing the
  model, dataset, and various optoins (learning rate, batch size, etc) to the
  constructor. You will then call the train() method to run the optimization
  procedure and train the model.
  
  After the train() method returns, model.params will contain the parameters
  that performed best on the validation set over the course of training.
  In addition, the instance variable solver.loss_history will contain a list
  of all losses encountered during training and the instance variables
  solver.train_acc_history and solver.val_acc_history will be lists containing
  the accuracies of the model on the training and validation set at each epoch.
  
  Example usage might look something like this:
  
  data = {
    'X_train': # training data
    'y_train': # training labels
    'X_val': # validation data
    'X_train': # validation labels
  }
  model = MyAwesomeModel(hidden_size=100, reg=10)
  solver = Solver(model, data,
                  update_rule='sgd',
                  optim_config={
                    'learning_rate': 1e-3,
                  },
                  lr_decay=0.95,
                  num_epochs=10, batch_size=100,
                  print_every=100)
  solver.train()


  A Solver works on a model object that must conform to the following API:

  - model.params must be a dictionary mapping string parameter names to numpy
    arrays containing parameter values.

  - model.loss(X, y) must be a function that computes training-time loss and
    gradients, and test-time classification scores, with the following inputs
    and outputs:

    Inputs:
    - X: Array giving a minibatch of input data of shape (N, d_1, ..., d_k)
    - y: Array of labels, of shape (N,) giving labels for X where y[i] is the
      label for X[i].

    Returns:
    If y is None, run a test-time forward pass and return:
    - scores: Array of shape (N, C) giving classification scores for X where
      scores[i, c] gives the score of class c for X[i].

    If y is not None, run a training time forward and backward pass and return
    a tuple of:
    - loss: Scalar giving the loss
    - grads: Dictionary with the same keys as self.params mapping parameter
      names to gradients of the loss with respect to those parameters.
  """

  def __init__(self, model, data, **kwargs):
    """
    Construct a new Solver instance.
    
    Required arguments:
    - model: A model object conforming to the API described above
    - data: A dictionary of training and validation data with the following:
      'X_train': Array of shape (N_train, d_1, ..., d_k) giving training images
      'X_val': Array of shape (N_val, d_1, ..., d_k) giving validation images
      'y_train': Array of shape (N_train,) giving labels for training images
      'y_val': Array of shape (N_val,) giving labels for validation images
      
    Optional arguments:
    - update_rule: A string giving the name of an update rule in optim.py.
      Default is 'sgd'.
    - optim_config: A dictionary containing hyperparameters that will be
      passed to the chosen update rule. Each update rule requires different
      hyperparameters (see optim.py) but all update rules require a
      'learning_rate' parameter so that should always be present.
    - lr_decay: A scalar for learning rate decay; after each epoch the learning
      rate is multiplied by this value.
    - batch_size: Size of minibatches used to compute loss and gradient during
      training.
    - num_epochs: The number of epochs to run for during training.
    - print_every: Integer; training losses will be printed every print_every
      iterations.
    - verbose: Boolean; if set to false then no output will be printed during
      training.
    """
    self.model = model
    self.X_train = data['X_train']
    self.y_train = data['y_train']
    self.X_val = data['X_val']
    self.y_val = data['y_val']
    
    # Unpack keyword arguments
    self.update_rule = kwargs.pop('update_rule', 'sgd')
    self.optim_config = kwargs.pop('optim_config', {})
    self.lr_decay = kwargs.pop('lr_decay', 1.0)
    self.batch_size = kwargs.pop('batch_size', 100)
    self.num_epochs = kwargs.pop('num_epochs', 10)

    self.print_every = kwargs.pop('print_every', 10)
    self.verbose = kwargs.pop('verbose', True)

    # Throw an error if there are extra keyword arguments
    if len(kwargs) > 0:
      extra = ', '.join('"%s"' % k for k in kwargs.keys())
      raise ValueError('Unrecognized arguments %s' % extra)

    # Make sure the update rule exists, then replace the string
    # name with the actual function
    if not hasattr(optim, self.update_rule):
      raise ValueError('Invalid update_rule "%s"' % self.update_rule)
    self.update_rule = getattr(optim, self.update_rule)

    self._reset()


  def _reset(self):
    """
    Set up some book-keeping variables for optimization. Don't call this
    manually.
    """
    # Set up some variables for book-keeping
    self.epoch = 0
    self.best_val_acc = 0
    self.best_params = {}
    self.loss_history = []
    self.train_acc_history = []
    self.val_acc_history = []

    # Make a deep copy of the optim_config for each parameter
    self.optim_configs = {}
    for p in self.model.params:
      d = {k: v for k, v in self.optim_config.iteritems()}
      self.optim_configs[p] = d


  def _step(self):
    """
    Make a single gradient update. This is called by train() and should not
    be called manually.
    """
    # Make a minibatch of training data
    num_train = self.X_train.shape[0]
    batch_mask = np.random.choice(num_train, self.batch_size)
    X_batch = self.X_train[batch_mask]
    y_batch = self.y_train[batch_mask]

    # Compute loss and gradient
    loss, grads = self.model.loss(X_batch, y_batch)
    self.loss_history.append(loss)

    # Perform a parameter update
    for p, w in self.model.params.iteritems():
      dw = grads[p]
      config = self.optim_configs[p]
      next_w, next_config = self.update_rule(w, dw, config)
      self.model.params[p] = next_w
      self.optim_configs[p] = next_config


  def check_accuracy(self, X, y, num_samples=None, batch_size=100):
    """
    Check accuracy of the model on the provided data.
    
    Inputs:
    - X: Array of data, of shape (N, d_1, ..., d_k)
    - y: Array of labels, of shape (N,)
    - num_samples: If not None, subsample the data and only test the model
      on num_samples datapoints.
    - batch_size: Split X and y into batches of this size to avoid using too
      much memory.
      
    Returns:
    - acc: Scalar giving the fraction of instances that were correctly
      classified by the model.
    """
    
    # Maybe subsample the data
    N = X.shape[0]
    if num_samples is not None and N > num_samples:
      mask = np.random.choice(N, num_samples)
      N = num_samples
      X = X[mask]
      y = y[mask]

    # Compute predictions in batches
    num_batches = N / batch_size
    if N % batch_size != 0:
      num_batches += 1
    y_pred = []
    for i in xrange(num_batches):
      start = i * batch_size
      end = (i + 1) * batch_size
      scores = self.model.loss(X[start:end])
      y_pred.append(np.argmax(scores, axis=1))
    y_pred = np.hstack(y_pred)
    acc = np.mean(y_pred == y)

    return acc


  def train(self):
    """
    Run optimization to train the model.
    """
    num_train = self.X_train.shape[0]
    iterations_per_epoch = max(num_train / self.batch_size, 1)
    num_iterations = self.num_epochs * iterations_per_epoch

    for t in xrange(num_iterations):
      self._step()

      # Maybe print training loss
      if self.verbose and t % self.print_every == 0:
        print '(Iteration %d / %d) loss: %f' % (
               t + 1, num_iterations, self.loss_history[-1])

      # At the end of every epoch, increment the epoch counter and decay the
      # learning rate.
      epoch_end = (t + 1) % iterations_per_epoch == 0
      if epoch_end:
        self.epoch += 1
        for k in self.optim_configs:
          self.optim_configs[k]['learning_rate'] *= self.lr_decay

      # Check train and val accuracy on the first iteration, the last
      # iteration, and at the end of each epoch.
      first_it = (t == 0)
      last_it = (t == num_iterations + 1)
      if first_it or last_it or epoch_end:
        train_acc = self.check_accuracy(self.X_train, self.y_train,
                                        num_samples=1000)
        val_acc = self.check_accuracy(self.X_val, self.y_val)
        self.train_acc_history.append(train_acc)
        self.val_acc_history.append(val_acc)

        if self.verbose:
          print '(Epoch %d / %d) train acc: %f; val_acc: %f' % (
                 self.epoch, self.num_epochs, train_acc, val_acc)

        # Keep track of the best model
        if val_acc > self.best_val_acc:
          self.best_val_acc = val_acc
          self.best_params = {}
          for k, v in self.model.params.iteritems():
            self.best_params[k] = v.copy()

    # At the end of training swap the best params into the model
    self.model.params = self.best_params

optim.py

import numpy as np

"""
This file implements various first-order update rules that are commonly used for
training neural networks. Each update rule accepts current weights and the
gradient of the loss with respect to those weights and produces the next set of
weights. Each update rule has the same interface:

def update(w, dw, config=None):

Inputs:
  - w: A numpy array giving the current weights.
  - dw: A numpy array of the same shape as w giving the gradient of the
    loss with respect to w.
  - config: A dictionary containing hyperparameter values such as learning rate,
    momentum, etc. If the update rule requires caching values over many
    iterations, then config will also hold these cached values.

Returns:
  - next_w: The next point after the update.
  - config: The config dictionary to be passed to the next iteration of the
    update rule.

NOTE: For most update rules, the default learning rate will probably not perform
well; however the default values of the other hyperparameters should work well
for a variety of different problems.

For efficiency, update rules may perform in-place updates, mutating w and
setting next_w equal to w.
"""


def sgd(w, dw, config=None):
  """
  Performs vanilla stochastic gradient descent.

  config format:
  - learning_rate: Scalar learning rate.
  """
  if config is None: config = {}
  config.setdefault('learning_rate', 1e-2)

  w -= config['learning_rate'] * dw
  return w, config


def sgd_momentum(w, dw, config=None):
  """
  Performs stochastic gradient descent with momentum.

  config format:
  - learning_rate: Scalar learning rate.
  - momentum: Scalar between 0 and 1 giving the momentum value.
    Setting momentum = 0 reduces to sgd.
  - velocity: A numpy array of the same shape as w and dw used to store a moving
    average of the gradients.
  """
  if config is None: config = {}
  config.setdefault('learning_rate', 1e-2)
  config.setdefault('momentum', 0.9)
  v = config.get('velocity', np.zeros_like(w))
  
  learning_rate = config['learning_rate']
  m = config['momentum']
    
  next_w = w
  #############################################################################
  # TODO: Implement the momentum update formula. Store the updated value in   #
  # the next_w variable. You should also use and update the velocity v.       #
  #############################################################################
  v  = m * v - learning_rate * dw
  next_w += v   
  #############################################################################
  #                             END OF YOUR CODE                              #
  #############################################################################
  config['velocity'] = v

  return next_w, config



def rmsprop(x, dx, config=None):
  """
  Uses the RMSProp update rule, which uses a moving average of squared gradient
  values to set adaptive per-parameter learning rates.

  config format:
  - learning_rate: Scalar learning rate.
  - decay_rate: Scalar between 0 and 1 giving the decay rate for the squared
    gradient cache.
  - epsilon: Small scalar used for smoothing to avoid dividing by zero.
  - cache: Moving average of second moments of gradients.
  """
  if config is None: config = {}
  config.setdefault('learning_rate', 1e-2)
  config.setdefault('decay_rate', 0.99)
  config.setdefault('epsilon', 1e-8)
  config.setdefault('cache', np.zeros_like(x))

  learning_rate = config['learning_rate']
  decay_rate = config['decay_rate']
  epsilon = config['epsilon']
  cache = config['cache']
  next_x = None
  #############################################################################
  # TODO: Implement the RMSprop update formula, storing the next value of x   #
  # in the next_x variable. Don't forget to update cache value stored in      #  
  # config['cache'].                                                          #
  #############################################################################
  cache = decay_rate * cache + (1 - decay_rate) * dx**2
  next_x = x - learning_rate * dx / (np.sqrt(cache) + epsilon)
  config['cache'] = cache   
  #############################################################################
  #                             END OF YOUR CODE                              #
  #############################################################################

  return next_x, config


def adam(x, dx, config=None):
  """
  Uses the Adam update rule, which incorporates moving averages of both the
  gradient and its square and a bias correction term.

  config format:
  - learning_rate: Scalar learning rate.
  - beta1: Decay rate for moving average of first moment of gradient.
  - beta2: Decay rate for moving average of second moment of gradient.
  - epsilon: Small scalar used for smoothing to avoid dividing by zero.
  - m: Moving average of gradient.
  - v: Moving average of squared gradient.
  - t: Iteration number.
  """
  if config is None: config = {}
  config.setdefault('learning_rate', 1e-3)
  config.setdefault('beta1', 0.9)
  config.setdefault('beta2', 0.999)
  config.setdefault('epsilon', 1e-8)
  config.setdefault('m', np.zeros_like(x))
  config.setdefault('v', np.zeros_like(x))
  config.setdefault('t', 0)
  
  learning_rate = config['learning_rate']
  beta1 = config['beta1']
  beta2 = config['beta2']
  epsilon = config['epsilon']
  m = config['m']
  v = config['v']
  t = config['t']   
    
  next_x = None
  #############################################################################
  # TODO: Implement the Adam update formula, storing the next value of x in   #
  # the next_x variable. Don't forget to update the m, v, and t variables     #
  # stored in config.                                                         #
  #############################################################################
  t = t + 1 
  m = beta1*m + (1-beta1)*dx
  v = beta2*v + (1-beta2)*(dx**2)
  m1 = m / (1 - beta1**t)
  v1 = v / (1 - beta2**t)
  next_x = x - learning_rate * m1 / (np.sqrt(v1) + epsilon)   
  config['m'] = m
  config['v'] = v
  config['t'] = t                                 
  #############################################################################
  #                             END OF YOUR CODE                              #
  #############################################################################
  
  return next_x, config

weight_scale = 1e-2
3 layer
learning_rate = 1e-2
Multilayer network

5 layer
learning_rate = 1e-2
weight_scale = 5e-2

train a good model
adam only, learning_rate 6e-4

Conv Net

import numpy as np

from cs231n.layers import *
from cs231n.fast_layers import *
from cs231n.layer_utils import *

# - [conv-relu-pool]XN - [affine]XM - [softmax or SVM]
class ThreeLayerConvNetWithConvReluPoolX2Naive(object):
    """
    A three-layer convolutional network with the following architecture:
  
    [conv - relu - 2x2 max pool]*2 - affine - relu - affine - softmax
  
    The network operates on minibatches of data that have shape (N, C, H, W)
    consisting of N images, each with height H and width W and with C input
    channels.
    """

    def __init__(self, input_dim=(3, 32, 32), num_filters=32, filter_size=7,
                 hidden_dim=100, num_classes=10, weight_scale=1e-3, reg=0.0,
                 dtype=np.float32):
        """
        Initialize a new network.
    
        Inputs:
        - input_dim: Tuple (C, H, W) giving size of input data
        - num_filters: Number of filters to use in the convolutional layer
        - filter_size: Size of filters to use in the convolutional layer
        - hidden_dim: Number of units to use in the fully-connected hidden layer
        - num_classes: Number of scores to produce from the final affine layer.
        - weight_scale: Scalar giving standard deviation for random initialization
          of weights.
        - reg: Scalar giving L2 regularization strength
        - dtype: numpy datatype to use for computation.
        """
        self.params = {}
        self.reg = reg
        self.dtype = dtype
        ############################################################################
        # TODO: Initialize weights and biases for the three-layer convolutional    #
        # network. Weights should be initialized from a Gaussian with standard     #
        # deviation equal to weight_scale; biases should be initialized to zero.   #
        # All weights and biases should be stored in the dictionary self.params.   #
        # Store weights and biases for the convolutional layer using the keys 'W1' #
        # and 'b1'; use keys 'W2' and 'b2' for the weights and biases of the       #
        # hidden affine layer, and keys 'W3' and 'b3' for the weights and biases   #
        # of the output affine layer.                                              #
        ############################################################################
        C, H, W = input_dim

        # first conv layer init
        F_CONV1  = num_filters
        HH_CONV1 = filter_size
        WW_CONV1 = filter_size
        W_CONV1 = np.random.randn(F_CONV1, C, HH_CONV1, WW_CONV1) * weight_scale
        b_CONV1 = np.zeros(F_CONV1)

        # second conv layer init
        F_CONV2  = num_filters
        HH_CONV2 = filter_size
        WW_CONV2 = filter_size
        W_CONV2 = np.random.randn(F_CONV2, F_CONV1, HH_CONV2, WW_CONV2) * weight_scale
        b_CONV2 = np.zeros(F_CONV2)

        # first affine layer
        H_AFF1 = hidden_dim
        W_AFF1 = np.random.randn(F_CONV2 * (H / 4) * (W / 4), H_AFF1) * weight_scale
        b_AFF1 = np.zeros(H_AFF1)
        # last affine layer
        H_LAST_AFF = num_classes
        W_LAST_AFF = np.random.randn(H_AFF1, H_LAST_AFF) * weight_scale
        b_LAST_AFF= np.zeros(H_LAST_AFF)

        # store them to self.prams for later use
        self.params['W_CONV1'] = W_CONV1
        self.params['b_CONV1'] = b_CONV1
        self.params['W_CONV2'] = W_CONV2
        self.params['b_CONV2'] = b_CONV2
        self.params['W_AFF1'] = W_AFF1
        self.params['b_AFF1'] = b_AFF1
        self.params['W_LAST_AFF'] = W_LAST_AFF
        self.params['b_LAST_AFF'] = b_LAST_AFF
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        for k, v in self.params.iteritems():
            #print "k",k,"v",v
            self.params[k] = v.astype(dtype)

    def loss(self, X, y=None):
        """
        Evaluate loss and gradient for the three-layer convolutional network.
    
        Input / output: Same API as TwoLayerNet in fc_net.py.
        """
        W_CONV1, b_CONV1 = self.params['W_CONV1'], self.params['b_CONV1']
        W_CONV2, b_CONV2 = self.params['W_CONV2'], self.params['b_CONV2']
        W_AFF1, b_AFF1 = self.params['W_AFF1'], self.params['b_AFF1']
        W_LAST_AFF, b_LAST_AFF = self.params['W_LAST_AFF'], self.params['b_LAST_AFF']

        # pass conv_param to the forward pass for the convolutional layer
        # 2 conv layer share same conv_params and pool_params
        # may vary them later
        filter_size = W_CONV1.shape[2]
        conv_param = {'stride': 1, 'pad': (filter_size - 1) / 2}

        # pass pool_param to the forward pass for the max-pooling layer
        pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2}

        scores = None
        ############################################################################
        # TODO: Implement the forward pass for the three-layer convolutional net,  #
        # computing the class scores for X and storing them in the scores          #
        # variable.                                                                #
        ############################################################################
        out_conv_relu_pool_1, cache_conv_relu_pool_1 = conv_relu_pool_forward(X, W_CONV1, b_CONV1, conv_param, pool_param)
        out_conv_relu_pool_2, cache_conv_relu_pool_2 = conv_relu_pool_forward(out_conv_relu_pool_1, W_CONV2, b_CONV2, conv_param, pool_param)

        # print "out_conv_relu_pool",out_conv_relu_pool.shape
        # print "W2",W2.shape
        # print "b2",b2.shape
        out_affine_relu_1, cache_affine_relu_1 = affine_relu_forward(out_conv_relu_pool_2, W_AFF1, b_AFF1)
        out, cache_affine_last = affine_forward(out_affine_relu_1, W_LAST_AFF, b_LAST_AFF)
        scores = out
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        if y is None:
            return scores

        loss, grads = 0, {}
        ############################################################################
        # TODO: Implement the backward pass for the three-layer convolutional net, #
        # storing the loss and gradients in the loss and grads variables. Compute  #
        # data loss using softmax, and make sure that grads[k] holds the gradients #
        # for self.params[k]. Don't forget to add L2 regularization!               #
        ############################################################################
        loss, dx = softmax_loss(scores, y)
        loss += 0.5 * self.reg * (np.sum(W_CONV1 * W_CONV1) + np.sum(W_CONV2 * W_CONV2) + np.sum(W_AFF1 * W_AFF1) + np.sum(W_LAST_AFF * W_LAST_AFF))

        # gradients, cool
        dout_last_aff, dw_last_aff, db_last_aff = affine_backward(dx, cache_affine_last)
        grads['W_LAST_AFF'] = dw_last_aff + self.reg * W_LAST_AFF
        grads['b_LAST_AFF'] = db_last_aff

        dout_first_affine_relu_aff1, dw_aff1, db_aff1 = affine_relu_backward(dout_last_aff, cache_affine_relu_1)
        grads['W_AFF1'] = dw_aff1 + self.reg * W_AFF1
        grads['b_AFF1'] = db_aff1

        dout_conv2, dw_conv2, db_conv2 = conv_relu_pool_backward(dout_first_affine_relu_aff1, cache_conv_relu_pool_2)
        grads['W_CONV2'] = dw_conv2 + self.reg * W_CONV2
        grads['b_CONV2'] = db_conv2

        dout_conv1, dw_conv1, db_conv1 = conv_relu_pool_backward(dout_conv2, cache_conv_relu_pool_1)
        grads['W_CONV1'] = dw_conv1 + self.reg * W_CONV1
        grads['b_CONV1'] = db_conv1
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        return loss, grads


# - [conv-relu-pool]XN - [affine]XM - [softmax or SVM]
class ThreeLayerConvNetWithConvReluPoolX2NaiveSpatialBN(object):
    """
    A three-layer convolutional network with the following architecture:

    [conv - relu - 2x2 max pool]*2 - affine - relu - affine - softmax

    The network operates on minibatches of data that have shape (N, C, H, W)
    consisting of N images, each with height H and width W and with C input
    channels.
    """

    def __init__(self, input_dim=(3, 32, 32), num_filters=32, filter_size=7,
                 hidden_dim=100, num_classes=10, weight_scale=1e-3, reg=0.0,
                 dtype=np.float32):
        """
        Initialize a new network.

        Inputs:
        - input_dim: Tuple (C, H, W) giving size of input data
        - num_filters: Number of filters to use in the convolutional layer
        - filter_size: Size of filters to use in the convolutional layer
        - hidden_dim: Number of units to use in the fully-connected hidden layer
        - num_classes: Number of scores to produce from the final affine layer.
        - weight_scale: Scalar giving standard deviation for random initialization
          of weights.
        - reg: Scalar giving L2 regularization strength
        - dtype: numpy datatype to use for computation.
        """
        self.params = {}
        self.reg = reg
        self.dtype = dtype
        ############################################################################
        # TODO: Initialize weights and biases for the three-layer convolutional    #
        # network. Weights should be initialized from a Gaussian with standard     #
        # deviation equal to weight_scale; biases should be initialized to zero.   #
        # All weights and biases should be stored in the dictionary self.params.   #
        # Store weights and biases for the convolutional layer using the keys 'W1' #
        # and 'b1'; use keys 'W2' and 'b2' for the weights and biases of the       #
        # hidden affine layer, and keys 'W3' and 'b3' for the weights and biases   #
        # of the output affine layer.                                              #
        ############################################################################
        self.bn_params = []
        if self.use_batchnorm:
            self.bn_params = [{'mode': 'train'} for i in range(0,2)]



        C, H, W = input_dim

        # first conv layer init
        F_CONV1 = num_filters
        HH_CONV1 = filter_size
        WW_CONV1 = filter_size
        W_CONV1 = np.random.randn(F_CONV1, C, HH_CONV1, WW_CONV1) * weight_scale
        b_CONV1 = np.zeros(F_CONV1)

        # second conv layer init
        F_CONV2 = num_filters
        HH_CONV2 = filter_size
        WW_CONV2 = filter_size
        W_CONV2 = np.random.randn(F_CONV2, F_CONV1, HH_CONV2, WW_CONV2) * weight_scale
        b_CONV2 = np.zeros(F_CONV2)

        # first affine layer
        H_AFF1 = hidden_dim
        W_AFF1 = np.random.randn(F_CONV2 * (H / 4) * (W / 4), H_AFF1) * weight_scale
        b_AFF1 = np.zeros(H_AFF1)
        # last affine layer
        H_LAST_AFF = num_classes
        W_LAST_AFF = np.random.randn(H_AFF1, H_LAST_AFF) * weight_scale
        b_LAST_AFF = np.zeros(H_LAST_AFF)

        # store them to self.prams for later use
        self.params['W_CONV1'] = W_CONV1
        self.params['b_CONV1'] = b_CONV1
        self.params['W_CONV2'] = W_CONV2
        self.params['b_CONV2'] = b_CONV2
        self.params['W_AFF1'] = W_AFF1
        self.params['b_AFF1'] = b_AFF1
        self.params['W_LAST_AFF'] = W_LAST_AFF
        self.params['b_LAST_AFF'] = b_LAST_AFF
        self.params["Gamma1"] = np.ones((1, F_CONV1))
        self.params["Beta1"] = np.zeros((1, F_CONV1))
        self.params["Gamma2"] = np.ones((1, F_CONV2))
        self.params["Beta2"] = np.zeros((1, F_CONV2))
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        for k, v in self.params.iteritems():
            # print "k",k,"v",v
            self.params[k] = v.astype(dtype)

    def loss(self, X, y=None):
        """
        Evaluate loss and gradient for the three-layer convolutional network.

        Input / output: Same API as TwoLayerNet in fc_net.py.
        """
        W_CONV1, b_CONV1 = self.params['W_CONV1'], self.params['b_CONV1']
        W_CONV2, b_CONV2 = self.params['W_CONV2'], self.params['b_CONV2']
        W_AFF1, b_AFF1 = self.params['W_AFF1'], self.params['b_AFF1']
        W_LAST_AFF, b_LAST_AFF = self.params['W_LAST_AFF'], self.params['b_LAST_AFF']
        Gamma1 = self.params["Gamma1"]
        Beta1 = self.params["Beta1"]
        Gamma2 = self.params["Gamma2"]
        Beta2 = self.params["Beta2"]
        # pass conv_param to the forward pass for the convolutional layer
        # 2 conv layer share same conv_params and pool_params
        # may vary them later
        filter_size = W_CONV1.shape[2]
        conv_param = {'stride': 1, 'pad': (filter_size - 1) / 2}

        # pass pool_param to the forward pass for the max-pooling layer
        pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2}

        scores = None
        ############################################################################
        # TODO: Implement the forward pass for the three-layer convolutional net,  #
        # computing the class scores for X and storing them in the scores          #
        # variable.                                                                #
        ############################################################################
        out_conv_relu_pool_1, cache_conv_relu_pool_1 = conv_relu_pool_forward(X, W_CONV1, b_CONV1, conv_param,
                                                                              pool_param)
        out_conv_relu_pool_2, cache_conv_relu_pool_2 = conv_relu_pool_forward(out_conv_relu_pool_1, W_CONV2, b_CONV2,
                                                                              conv_param, pool_param)

        # print "out_conv_relu_pool",out_conv_relu_pool.shape
        # print "W2",W2.shape
        # print "b2",b2.shape
        out_affine_relu_1, cache_affine_relu_1 = affine_relu_forward(out_conv_relu_pool_2, W_AFF1, b_AFF1)
        out, cache_affine_last = affine_forward(out_affine_relu_1, W_LAST_AFF, b_LAST_AFF)
        scores = out
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        if y is None:
            return scores

        loss, grads = 0, {}
        ############################################################################
        # TODO: Implement the backward pass for the three-layer convolutional net, #
        # storing the loss and gradients in the loss and grads variables. Compute  #
        # data loss using softmax, and make sure that grads[k] holds the gradients #
        # for self.params[k]. Don't forget to add L2 regularization!               #
        ############################################################################
        loss, dx = softmax_loss(scores, y)
        loss += 0.5 * self.reg * (
        np.sum(W_CONV1 * W_CONV1) + np.sum(W_CONV2 * W_CONV2) + np.sum(W_AFF1 * W_AFF1)) + np.sum(
            W_LAST_AFF * W_LAST_AFF)

        # gradients, cool
        dout_last_aff, dw_last_aff, db_last_aff = affine_backward(dx, cache_affine_last)
        grads['W_LAST_AFF'] = dw_last_aff + self.reg * W_LAST_AFF
        grads['b_LAST_AFF'] = db_last_aff

        dout_first_affine_relu_aff1, dw_aff1, db_aff1 = affine_relu_backward(dout_last_aff, cache_affine_relu_1)
        grads['W_AFF1'] = dw_aff1 + self.reg * W_AFF1
        grads['b_AFF1'] = db_aff1

        dout_conv2, dw_conv2, db_conv2 = conv_relu_pool_backward(dout_first_affine_relu_aff1, cache_conv_relu_pool_2)
        grads['W_CONV2'] = dw_conv2 + self.reg * W_CONV2
        grads['b_CONV2'] = db_conv2

        dout_conv1, dw_conv1, db_conv1 = conv_relu_pool_backward(dout_conv2, cache_conv_relu_pool_1)
        grads['W_CONV1'] = dw_conv1 + self.reg * W_CONV1
        grads['b_CONV1'] = db_conv1
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        return loss, grads
import numpy as np

from cs231n.layers import *
from cs231n.fast_layers import *
from cs231n.layer_utils import *

# - [conv-relu-pool]XN - [affine]XM - [softmax or SVM]
class ThreeLayerConvNetWithConvReluPoolXNNaive(object):
    """
    A three-layer convolutional network with the following architecture:
  
    [conv - relu - 2x2 max pool]*2 - affine - relu - affine - softmax
  
    The network operates on minibatches of data that have shape (N, C, H, W)
    consisting of N images, each with height H and width W and with C input
    channels.
    """

    def __init__(self, input_dim=(3, 32, 32), num_filters_list=[32], filter_sizes=[7],
                 hidden_dim=100, num_classes=10, weight_scale=1e-3, reg=0.0,
                 dtype=np.float32, conv_relu_pool_num=2):
        """
        Initialize a new network.
    
        Inputs:
        - input_dim: Tuple (C, H, W) giving size of input data
        - num_filters: Number of filters to use in the convolutional layer
        - filter_size: Size of filters to use in the convolutional layer
        - hidden_dim: Number of units to use in the fully-connected hidden layer
        - num_classes: Number of scores to produce from the final affine layer.
        - weight_scale: Scalar giving standard deviation for random initialization
          of weights.
        - reg: Scalar giving L2 regularization strength
        - dtype: numpy datatype to use for computation.
        """
        self.params = {}
        self.reg = reg
        self.dtype = dtype
        ############################################################################
        # TODO: Initialize weights and biases for the three-layer convolutional    #
        # network. Weights should be initialized from a Gaussian with standard     #
        # deviation equal to weight_scale; biases should be initialized to zero.   #
        # All weights and biases should be stored in the dictionary self.params.   #
        # Store weights and biases for the convolutional layer using the keys 'W1' #
        # and 'b1'; use keys 'W2' and 'b2' for the weights and biases of the       #
        # hidden affine layer, and keys 'W3' and 'b3' for the weights and biases   #
        # of the output affine layer.                                              #
        ############################################################################
        C, H, W = input_dim
        self.conv_relu_pool_num = len(num_filters_list)
        self.filter_sizes = filter_sizes
        # first conv layer init
        fan_in = None
        fan_out = C
        for i in range(0, self.conv_relu_pool_num):
            fan_in = num_filters_list[i]
            self.params["W_CONV" + str(i+1)] = np.random.randn(fan_in, fan_out, filter_sizes[i], filter_sizes[i]) * weight_scale
            self.params["b_CONV" + str(i+1)] = np.zeros(fan_in)
            fan_out = num_filters_list[i]

        # first affine layer
        H_AFF1 = hidden_dim
        W_AFF1 = np.random.randn(fan_in * (H / (2 ** len(num_filters_list))) * (W / (2 ** len(num_filters_list))), H_AFF1) * weight_scale
        b_AFF1 = np.zeros(H_AFF1)
        # last affine layer
        H_LAST_AFF = num_classes
        W_LAST_AFF = np.random.randn(H_AFF1, H_LAST_AFF) * weight_scale
        b_LAST_AFF= np.zeros(H_LAST_AFF)

        # store non-conv params self.prams for later use
        self.params['W_AFF1'] = W_AFF1
        self.params['b_AFF1'] = b_AFF1
        self.params['W_LAST_AFF'] = W_LAST_AFF
        self.params['b_LAST_AFF'] = b_LAST_AFF
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        for k, v in self.params.iteritems():
            #print "k",k,"v",v
            self.params[k] = v.astype(dtype)

    def loss(self, X, y=None):
        """
        Evaluate loss and gradient for the three-layer convolutional network.
    
        Input / output: Same API as TwoLayerNet in fc_net.py.
        """
        W_AFF1, b_AFF1 = self.params['W_AFF1'], self.params['b_AFF1']
        W_LAST_AFF, b_LAST_AFF = self.params['W_LAST_AFF'], self.params['b_LAST_AFF']

        # pass conv_param to the forward pass for the convolutional layer
        # 2 conv layer share same conv_params and pool_params
        # may vary them later
        conv_params = []
        for i in range(0, self.conv_relu_pool_num):
            conv_params.append({'stride': 1, 'pad': (self.filter_sizes[i] - 1) / 2})

        # pass pool_param to the forward pass for the max-pooling layer
        pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2}

        scores = None
        ############################################################################
        # TODO: Implement the forward pass for the three-layer convolutional net,  #
        # computing the class scores for X and storing them in the scores          #
        # variable.                                                                #
        ############################################################################
        caches_conv_relu_pool = []
        input = X
        for i in range(0, self.conv_relu_pool_num):
            out_conv_relu_pool, cache_conv_relu_pool = conv_relu_pool_forward(input, self.params["W_CONV" + str(i+1)], self.params["b_CONV" + str(i+1)], conv_params[i], pool_param)
            input = out_conv_relu_pool
            caches_conv_relu_pool.append(cache_conv_relu_pool)
        # print "out_conv_relu_pool",out_conv_relu_pool.shape
        # print "W2",W2.shape
        # print "b2",b2.shape
        out_affine_relu_1, cache_affine_relu_1 = affine_relu_forward(out_conv_relu_pool, W_AFF1, b_AFF1)
        out, cache_affine_last = affine_forward(out_affine_relu_1, W_LAST_AFF, b_LAST_AFF)
        scores = out
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        if y is None:
            return scores

        loss, grads = 0, {}
        ############################################################################
        # TODO: Implement the backward pass for the three-layer convolutional net, #
        # storing the loss and gradients in the loss and grads variables. Compute  #
        # data loss using softmax, and make sure that grads[k] holds the gradients #
        # for self.params[k]. Don't forget to add L2 regularization!               #
        ############################################################################
        loss, dx = softmax_loss(scores, y)
        regs_conv_relu_pool = 0.0
        for i in range(0, self.conv_relu_pool_num):
            regs_conv_relu_pool += 0.5 * self.reg * (np.sum(self.params["W_CONV" + str(i+1)]))
        loss += regs_conv_relu_pool + 0.5 * self.reg * (np.sum(W_AFF1 * W_AFF1) + np.sum(W_LAST_AFF * W_LAST_AFF))

        # gradients, cool
        dout_last_aff, dw_last_aff, db_last_aff = affine_backward(dx, cache_affine_last)
        grads['W_LAST_AFF'] = dw_last_aff + self.reg * W_LAST_AFF
        grads['b_LAST_AFF'] = db_last_aff

        dout_first_affine_relu_aff1, dw_aff1, db_aff1 = affine_relu_backward(dout_last_aff, cache_affine_relu_1)
        grads['W_AFF1'] = dw_aff1 + self.reg * W_AFF1
        grads['b_AFF1'] = db_aff1

        dout = dout_first_affine_relu_aff1
        for i in range(self.conv_relu_pool_num-1, -1, -1):
            dout, dw, db = conv_relu_pool_backward(dout, caches_conv_relu_pool[i])
            grads['W_CONV' + str(i+1)] = dw + self.reg * self.params["W_CONV" + str(i+1)]
            grads['b_CONV' + str(i+1)] = db
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        return loss, grads


# - [conv-relu-pool]XN - [affine]XM - [softmax or SVM]
class ThreeLayerConvNetWithConvReluPoolX2Naive2(object):
    """
    A three-layer convolutional network with the following architecture:

    [conv - relu - 2x2 max pool]*2 - affine - relu - affine - softmax

    The network operates on minibatches of data that have shape (N, C, H, W)
    consisting of N images, each with height H and width W and with C input
    channels.
    """

    def __init__(self, input_dim=(3, 32, 32), num_filters=32, filter_size=7,
                 hidden_dim=100, num_classes=10, weight_scale=1e-3, reg=0.0,
                 dtype=np.float32):
        """
        Initialize a new network.

        Inputs:
        - input_dim: Tuple (C, H, W) giving size of input data
        - num_filters: Number of filters to use in the convolutional layer
        - filter_size: Size of filters to use in the convolutional layer
        - hidden_dim: Number of units to use in the fully-connected hidden layer
        - num_classes: Number of scores to produce from the final affine layer.
        - weight_scale: Scalar giving standard deviation for random initialization
          of weights.
        - reg: Scalar giving L2 regularization strength
        - dtype: numpy datatype to use for computation.
        """
        self.params = {}
        self.reg = reg
        self.dtype = dtype
        ############################################################################
        # TODO: Initialize weights and biases for the three-layer convolutional    #
        # network. Weights should be initialized from a Gaussian with standard     #
        # deviation equal to weight_scale; biases should be initialized to zero.   #
        # All weights and biases should be stored in the dictionary self.params.   #
        # Store weights and biases for the convolutional layer using the keys 'W1' #
        # and 'b1'; use keys 'W2' and 'b2' for the weights and biases of the       #
        # hidden affine layer, and keys 'W3' and 'b3' for the weights and biases   #
        # of the output affine layer.                                              #
        ############################################################################
        self.bn_params = []
        if self.use_batchnorm:
            self.bn_params = [{'mode': 'train'} for i in range(0,2)]



        C, H, W = input_dim

        # first conv layer init
        F_CONV1 = num_filters
        HH_CONV1 = filter_size
        WW_CONV1 = filter_size
        W_CONV1 = np.random.randn(F_CONV1, C, HH_CONV1, WW_CONV1) * weight_scale
        b_CONV1 = np.zeros(F_CONV1)

        # second conv layer init
        F_CONV2 = num_filters * 2
        HH_CONV2 = filter_size
        WW_CONV2 = filter_size
        W_CONV2 = np.random.randn(F_CONV2, F_CONV1, HH_CONV2, WW_CONV2) * weight_scale
        b_CONV2 = np.zeros(F_CONV2)

        # first affine layer
        H_AFF1 = hidden_dim
        W_AFF1 = np.random.randn(F_CONV2 * (H / 4) * (W / 4), H_AFF1) * weight_scale
        b_AFF1 = np.zeros(H_AFF1)
        # last affine layer
        H_LAST_AFF = num_classes
        W_LAST_AFF = np.random.randn(H_AFF1, H_LAST_AFF) * weight_scale
        b_LAST_AFF = np.zeros(H_LAST_AFF)

        # store them to self.prams for later use
        self.params['W_CONV1'] = W_CONV1
        self.params['b_CONV1'] = b_CONV1
        self.params['W_CONV2'] = W_CONV2
        self.params['b_CONV2'] = b_CONV2
        self.params['W_AFF1'] = W_AFF1
        self.params['b_AFF1'] = b_AFF1
        self.params['W_LAST_AFF'] = W_LAST_AFF
        self.params['b_LAST_AFF'] = b_LAST_AFF
        self.params["Gamma1"] = np.ones((1, F_CONV1))
        self.params["Beta1"] = np.zeros((1, F_CONV1))
        self.params["Gamma2"] = np.ones((1, F_CONV2))
        self.params["Beta2"] = np.zeros((1, F_CONV2))
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        for k, v in self.params.iteritems():
            # print "k",k,"v",v
            self.params[k] = v.astype(dtype)

    def loss(self, X, y=None):
        """
        Evaluate loss and gradient for the three-layer convolutional network.

        Input / output: Same API as TwoLayerNet in fc_net.py.
        """
        W_CONV1, b_CONV1 = self.params['W_CONV1'], self.params['b_CONV1']
        W_CONV2, b_CONV2 = self.params['W_CONV2'], self.params['b_CONV2']
        W_AFF1, b_AFF1 = self.params['W_AFF1'], self.params['b_AFF1']
        W_LAST_AFF, b_LAST_AFF = self.params['W_LAST_AFF'], self.params['b_LAST_AFF']
        Gamma1 = self.params["Gamma1"]
        Beta1 = self.params["Beta1"]
        Gamma2 = self.params["Gamma2"]
        Beta2 = self.params["Beta2"]
        # pass conv_param to the forward pass for the convolutional layer
        # 2 conv layer share same conv_params and pool_params
        # may vary them later
        filter_size = W_CONV1.shape[2]
        conv_param = {'stride': 1, 'pad': (filter_size - 1) / 2}

        # pass pool_param to the forward pass for the max-pooling layer
        pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2}

        scores = None
        ############################################################################
        # TODO: Implement the forward pass for the three-layer convolutional net,  #
        # computing the class scores for X and storing them in the scores          #
        # variable.                                                                #
        ############################################################################
        out_conv_relu_pool_1, cache_conv_relu_pool_1 = conv_relu_pool_forward(X, W_CONV1, b_CONV1, conv_param,
                                                                              pool_param)
        out_conv_relu_pool_2, cache_conv_relu_pool_2 = conv_relu_pool_forward(out_conv_relu_pool_1, W_CONV2, b_CONV2,
                                                                              conv_param, pool_param)

        # print "out_conv_relu_pool",out_conv_relu_pool.shape
        # print "W2",W2.shape
        # print "b2",b2.shape
        out_affine_relu_1, cache_affine_relu_1 = affine_relu_forward(out_conv_relu_pool_2, W_AFF1, b_AFF1)
        out, cache_affine_last = affine_forward(out_affine_relu_1, W_LAST_AFF, b_LAST_AFF)
        scores = out
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        if y is None:
            return scores

        loss, grads = 0, {}
        ############################################################################
        # TODO: Implement the backward pass for the three-layer convolutional net, #
        # storing the loss and gradients in the loss and grads variables. Compute  #
        # data loss using softmax, and make sure that grads[k] holds the gradients #
        # for self.params[k]. Don't forget to add L2 regularization!               #
        ############################################################################
        loss, dx = softmax_loss(scores, y)
        loss += 0.5 * self.reg * (
        np.sum(W_CONV1 * W_CONV1) + np.sum(W_CONV2 * W_CONV2) + np.sum(W_AFF1 * W_AFF1)) + np.sum(
            W_LAST_AFF * W_LAST_AFF)

        # gradients, cool
        dout_last_aff, dw_last_aff, db_last_aff = affine_backward(dx, cache_affine_last)
        grads['W_LAST_AFF'] = dw_last_aff + self.reg * W_LAST_AFF
        grads['b_LAST_AFF'] = db_last_aff

        dout_first_affine_relu_aff1, dw_aff1, db_aff1 = affine_relu_backward(dout_last_aff, cache_affine_relu_1)
        grads['W_AFF1'] = dw_aff1 + self.reg * W_AFF1
        grads['b_AFF1'] = db_aff1

        dout_conv2, dw_conv2, db_conv2 = conv_relu_pool_backward(dout_first_affine_relu_aff1, cache_conv_relu_pool_2)
        grads['W_CONV2'] = dw_conv2 + self.reg * W_CONV2
        grads['b_CONV2'] = db_conv2

        dout_conv1, dw_conv1, db_conv1 = conv_relu_pool_backward(dout_conv2, cache_conv_relu_pool_1)
        grads['W_CONV1'] = dw_conv1 + self.reg * W_CONV1
        grads['b_CONV1'] = db_conv1
        ############################################################################
        #                             END OF YOUR CODE                             #
        ############################################################################

        return loss, grads

//last part
# Train a really good model on CIFAR-10
from cs231n.classifiers.convnet import *
from cs231n.classifiers.convnet_fancier import *
model = ThreeLayerConvNetWithConvReluPoolXNNaive(num_filters_list = [32,32], filter_sizes=[3,3], weight_scale=0.001, hidden_dim=500, reg=0.0005)
solver = Solver(model, data,
num_epochs=20, batch_size=100,
update_rule='adam',
optim_config={
'learning_rate': 1e-3,
},
verbose=True, print_every=20)
solver.train()

plt.subplot(2, 1, 1)
plt.plot(solver.train_acc_history)
plt.title('train_acc_history')

plt.subplot(2, 1, 2)
plt.plot(solver.val_acc_history)
plt.title('val_acc_history')
plt.show()


xiaolizi
转载须以超链接形式标明文章原始出处和作者信息

发表评论