"""Computes sigmoid cross entropy given `logits`.

with ops.op_scope([logits, targets], name, "logistic_loss") as name:

logits = ops.convert_to_tensor(logits, name="logits")

targets = ops.convert_to_tensor(targets, name="targets")

# The logistic loss formula from above is

# x - x * z + log(1 + exp(-x))

# For x < 0, a more numerically stable formula is

# -x * z + log(1 + exp(x))

# To avoid branching, we use the combined version

# max(x, 0) - x * z + log(1 + exp(-abs(x)))

return math_ops.add(nn_ops.relu(logits) - logits * targets,

math_ops.log(1 + math_ops.exp(-math_ops.abs(logits))),

"""Computes matmul(x, weights) + biases.

with ops.op_scope([x, weights, biases], name, "xw_plus_b") as name:

x = ops.convert_to_tensor(x, name="x")

weights = ops.convert_to_tensor(weights, name="weights")

biases = ops.convert_to_tensor(biases, name="biases")

mm = math_ops.matmul(x, weights)

"""Computes Relu(x * weight + biases).

with ops.op_scope([x, weights, biases], name, "relu_layer") as name:

x = ops.convert_to_tensor(x, name="x")

weights = ops.convert_to_tensor(weights, name="weights")

biases = ops.convert_to_tensor(biases, name="biases")

xw_plus_b = nn_ops.bias_add(math_ops.matmul(x, weights), biases)

"""Normalizes along dimension `dim` using an L2 norm.

with ops.op_scope([x], name, "l2_normalize") as name:

x = ops.convert_to_tensor(x, name="x")

square_sum = math_ops.reduce_sum(math_ops.square(x), [dim], keep_dims=True)

x_inv_norm = math_ops.rsqrt(math_ops.maximum(square_sum, epsilon))

"""Returns the fraction of zeros in `value`.

with ops.op_scope([value], name, "zero_fraction"):

value = ops.convert_to_tensor(value, name="value")

zero = constant_op.constant(0, dtype=value.dtype, name="zero")

return math_ops.reduce_mean(math_ops.cast(math_ops.equal(value, zero),

"""Computes dropout.

with ops.op_scope([x], name, "dropout") as name:

x = ops.convert_to_tensor(x, name="x")

if isinstance(keep_prob, float) and not(0 < keep_prob <= 1):

raise ValueError("keep_prob must be a scalar tensor or a float in the "

"range (0, 1], got %g" % keep_prob)

keep_prob = ops.convert_to_tensor(

keep_prob, dtype=x.dtype, name="keep_prob")

keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())

noise_shape = noise_shape or array_ops.shape(x)

# uniform [keep_prob, 1.0 + keep_prob)

random_tensor = keep_prob

random_tensor += random_ops.random_uniform(

noise_shape, seed=seed, dtype=x.dtype)

# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)

binary_tensor = math_ops.floor(random_tensor)

ret = x * math_ops.inv(keep_prob) * binary_tensor

ret.set_shape(x.get_shape())

"""Depthwise 2-D convolution.

with ops.op_scope([input, filter], name, "depthwise") as name:

input = ops.convert_to_tensor(input, name="tensor_in")

filter = ops.convert_to_tensor(filter, name="filter_in")

# A shape is required to statically compute the number of separable filters.

if filter.get_shape().ndims is not None:

assert len(filter.get_shape()) == 4

in_channels = filter.get_shape()[2]

# Sanity checks, if shape information is available for the inputs.

if input.get_shape().ndims is not None:

assert len(input.get_shape()) == 4

assert input.get_shape()[3] == in_channels, (

"Mismatched input depth %d and number of depthwise filters %d." % (

input.get_shape()[3].value, in_channels))

else:

assert input.get_shape().ndims is not None, (

"Either tensor must provide static shape information.")

assert input.get_shape().ndims == 4

in_channels = input.get_shape()[3]

if in_channels == 1:

return nn_ops.conv2d(input, filter, strides, padding, name=name)

else:

# Create one separate convolution per channel.

convs = []

for channel in xrange(in_channels):

with ops.name_scope("depth%d" % channel) as channel_scope:

t_in = array_ops.slice(input, [0, 0, 0, channel], [-1, -1, -1, 1],

name="slice_inputs")

f_in = array_ops.slice(filter, [0, 0, channel, 0], [-1, -1, 1, -1],

name="slice_params")

convs.append(nn_ops.conv2d(t_in, f_in,

strides, padding, name=channel_scope))

# Concatenate the per-channel convolutions along the channel dimension.

padding,

name=None):

"""2-D convolution with separable filters.

with ops.op_scope([input, depthwise_filter, pointwise_filter],

name, "separable_conv2d") as name:

input = ops.convert_to_tensor(input, name="tensor_in")

depthwise_filter = ops.convert_to_tensor(depthwise_filter,

name="depthwise_filter")

pointwise_filter = ops.convert_to_tensor(pointwise_filter,

name="pointwise_filter")

if pointwise_filter.get_shape().ndims is not None:

assert len(pointwise_filter.get_shape()) == 4

assert pointwise_filter.get_shape()[0] == 1

assert pointwise_filter.get_shape()[1] == 1

if depthwise_filter.get_shape().ndims and input.get_shape().ndims:

channel_multiplier = depthwise_filter.get_shape()[3]

in_channels = input.get_shape()[3]

out_channels = pointwise_filter.get_shape()[3]

# This would mean the separable convolutions is over-parametrized.

assert channel_multiplier * in_channels < out_channels

# The layout of the ops in the graph are expected to be as follows:

# separable_conv2d // Conv2D op corresponding to the pointwise conv.

# separable_conv2d/depthwise // Concat op for the deptwise outputs.

# separable_conv2d/depthwise/depth0 // Conv2D op for depth 0

# separable_conv2d/depthwise/depth1 // Conv2D op for depth 1

# separable_conv2d/depthwise/depth2 // Conv2D op for depth 2

depthwise = depthwise_conv2d(input, depthwise_filter, strides,

padding, name="depthwise")

return nn_ops.conv2d(depthwise, pointwise_filter, [1, 1, 1, 1],

"""Calculate the mean and variance of `x`.

with ops.op_scope([x, axes], name, "moments"):

x = ops.convert_to_tensor(x, name="x")

x_shape = x.get_shape()

if all(x_shape[d].value is not None for d in axes):

# The shape is known in the relevant axes, so we can statically

# compute the divisor.

divisor = 1.0

for d in set(axes):

divisor *= x.get_shape()[d].value

divisor = constant_op.constant(1.0 / divisor, x.dtype, name="divisor")

else:

divisor = constant_op.constant(1.0, dtype=x.dtype)

x_dynamic_shape = array_ops.shape(x)

for d in set(axes):

divisor *= math_ops.cast(x_dynamic_shape[d], x.dtype)

divisor = math_ops.inv(divisor, name="divisor")

axes = constant_op.constant(axes, name="axes")

# Note: We do not use Mean here because it is very slow on GPU.

# Note 2: The expression below is potentially more stable.

# It is however a bit slower and stability doesn't appear to be an issue.

# mean = math_ops.reduce_sum(math_ops.mul(x, divisor), axes, name="mean")

# var = math_ops.reduce_sum(math_ops.mul(math_ops.square(x - mean),

# divisor), axes,

# name="variance")

mean = math_ops.mul(math_ops.reduce_sum(x, axes), divisor, name="mean")

var = math_ops.mul(math_ops.reduce_sum(math_ops.square(x - mean), axes),

divisor, name="variance")

"""Returns a vector summing up each row of the matrix x."""

# _sum_rows(x) is equivalent to math_ops.reduce_sum(x, 1) when x is

# a matrix. The gradient of _sum_rows(x) is more efficient than

# reduce_sum(x, 1)'s gradient in today's implementation. Therefore,

# we use _sum_rows(x) in the nce_loss() computation since the loss

# is mostly used for training.

cols = array_ops.shape(x)[1]

ones_shape = array_ops.pack([cols, 1])

ones = array_ops.ones(ones_shape, x.dtype)

num_classes, num_true=1,

sampled_values=None,

subtract_log_q=True,

remove_accidental_hits=False,

name=None):

"""Helper function for nce_loss and sampled_softmax_loss functions.

if not isinstance(weights, list):

weights = [weights]

with ops.op_scope(

weights + [biases, inputs, labels], name, "compute_sampled_logits"):

if labels.dtype != dtypes.int64:

labels = math_ops.cast(labels, dtypes.int64)

labels_flat = array_ops.reshape(labels, [-1])

# Sample the negative labels.

# sampled shape: num_sampled vector

# true_expected_count shape = [batch_size, 1]

# sampled_expected_count shape = num_sampled vector

if sampled_values is None:

sampled_values = candidate_sampling_ops.log_uniform_candidate_sampler(

true_classes=labels,

num_true=num_true,

num_sampled=num_sampled,

unique=True,

range_max=num_classes)

# NOTE: pylint cannot tell that 'sampled_values' is a sequence

# pylint: disable=unpacking-non-sequence

sampled, true_expected_count, sampled_expected_count = sampled_values

# pylint: enable=unpacking-non-sequence

# weights shape is [num_classes, dim]

# labels_flat is a [batch_size * num_true] vector

# true_w shape is [batch_size * num_true, dim]

# true_b is a [batch_size * num_true] vector

true_w = embedding_ops.embedding_lookup(weights, labels_flat)

true_b = embedding_ops.embedding_lookup(biases, labels_flat)

# inputs shape is [batch_size, dim]

# true_w shape is [batch_size * num_true, dim]

# row_wise_dots is [batch_size, num_true, dim]

dim = array_ops.shape(true_w)[1:2]

new_true_w_shape = array_ops.concat(0, [[-1, num_true], dim])

row_wise_dots = math_ops.mul(

array_ops.expand_dims(inputs, 1),

array_ops.reshape(true_w, new_true_w_shape))

# We want the row-wise dot plus biases which yields a

# [batch_size, num_true] tensor of true_logits.

dots_as_matrix = array_ops.reshape(row_wise_dots,

array_ops.concat(0, [[-1], dim]))

true_logits = array_ops.reshape(_sum_rows(dots_as_matrix), [-1, num_true])

true_b = array_ops.reshape(true_b, [-1, num_true])

true_logits += true_b

# Lookup weights and biases for sampled labels.

# sampled is a num_sampled int vector

# sampled_w shape is [num_sampled, dim]

# sampled_b is a num_sampled float vector

sampled_w = embedding_ops.embedding_lookup(weights, sampled)

sampled_b = embedding_ops.embedding_lookup(biases, sampled)

# inputs has shape [batch_size, dim]

# sampled_w has shape [num_sampled, dim]

# sampled_b has shape [num_sampled]

# Apply X*W'+B, which yields [batch_size, num_sampled]

sampled_logits = math_ops.matmul(inputs,

sampled_w,

transpose_b=True) + sampled_b

if remove_accidental_hits:

acc_hits = candidate_sampling_ops.compute_accidental_hits(

labels, sampled, num_true=num_true)

acc_indices, acc_ids, acc_weights = acc_hits

# This is how SparseToDense expects the indices.

acc_indices_2d = array_ops.reshape(acc_indices, [-1, 1])

acc_ids_2d_int32 = array_ops.reshape(math_ops.cast(

acc_ids, dtypes.int32), [-1, 1])

sparse_indices = array_ops.concat(

1, [acc_indices_2d, acc_ids_2d_int32], "sparse_indices")

# Create sampled_logits_shape = [batch_size, num_sampled]

sampled_logits_shape = array_ops.concat(

0,

[array_ops.shape(labels)[:1], array_ops.expand_dims(num_sampled, 0)])

sampled_logits += sparse_ops.sparse_to_dense(

sparse_indices, sampled_logits_shape, acc_weights, 0.0)

if subtract_log_q:

# Subtract log of Q(l), prior probability that l appears in sampled.

true_logits -= math_ops.log(true_expected_count)

sampled_logits -= math_ops.log(sampled_expected_count)

# Construct output logits and labels. The true labels/logits start at col 0.

out_logits = array_ops.concat(1, [true_logits, sampled_logits])

# true_logits is a float tensor, ones_like(true_logits) is a float tensor

# of ones. We then divide by num_true to ensure the per-example labels sum

# to 1.0, i.e. form a proper probability distribution.

out_labels = array_ops.concat(

1, [array_ops.ones_like(true_logits) / num_true,

array_ops.zeros_like(sampled_logits)])

num_true=1,

sampled_values=None,

remove_accidental_hits=False,

name="nce_loss"):

"""Computes and returns the noise-contrastive estimation training loss.

logits, labels = _compute_sampled_logits(

weights, biases, inputs, labels, num_sampled, num_classes,

num_true=num_true,

sampled_values=sampled_values,

subtract_log_q=True,

remove_accidental_hits=remove_accidental_hits,

name=name)

sampled_losses = sigmoid_cross_entropy_with_logits(logits,

labels,

name="sampled_losses")

# sampled_losses is batch_size x {true_loss, sampled_losses...}

# We sum out true and sampled losses.

num_classes, num_true=1,

sampled_values=None,

remove_accidental_hits=True,

name="sampled_softmax_loss"):

"""Computes and returns the sampled softmax training loss.

logits, labels = _compute_sampled_logits(

weights, biases, inputs, labels, num_sampled, num_classes,

num_true=num_true,

sampled_values=sampled_values,

subtract_log_q=True,

remove_accidental_hits=remove_accidental_hits,

name=name)

sampled_losses = nn_ops.softmax_cross_entropy_with_logits(logits, labels)

# sampled_losses is a batch_size vector.