scope=None):

"""RNN decoder for the sequence-to-sequence model.

with vs.variable_scope(scope or "rnn_decoder"):

states = [initial_state]

outputs = []

prev = None

for i in xrange(len(decoder_inputs)):

inp = decoder_inputs[i]

if loop_function is not None and prev is not None:

with vs.variable_scope("loop_function", reuse=True):

# We do not propagate gradients over the loop function.

inp = array_ops.stop_gradient(loop_function(prev, i))

if i > 0:

vs.get_variable_scope().reuse_variables()

output, new_state = cell(inp, states[-1])

outputs.append(output)

states.append(new_state)

if loop_function is not None:

prev = array_ops.stop_gradient(output)

encoder_inputs, decoder_inputs, cell, dtype=dtypes.float32, scope=None):

"""Basic RNN sequence-to-sequence model.

with vs.variable_scope(scope or "basic_rnn_seq2seq"):

_, enc_states = rnn.rnn(cell, encoder_inputs, dtype=dtype)

loop_function=None, dtype=dtypes.float32, scope=None):

"""RNN sequence-to-sequence model with tied encoder and decoder parameters.

with vs.variable_scope("combined_tied_rnn_seq2seq"):

scope = scope or "tied_rnn_seq2seq"

_, enc_states = rnn.rnn(

cell, encoder_inputs, dtype=dtype, scope=scope)

vs.get_variable_scope().reuse_variables()

return rnn_decoder(decoder_inputs, enc_states[-1], cell,

output_projection=None, feed_previous=False,

scope=None):

"""RNN decoder with embedding and a pure-decoding option.

if output_projection is not None:

proj_weights = ops.convert_to_tensor(

output_projection[0], dtype=dtypes.float32)

proj_weights.get_shape().assert_is_compatible_with([cell.output_size,

num_symbols])

proj_biases = ops.convert_to_tensor(

output_projection[1], dtype=dtypes.float32)

proj_biases.get_shape().assert_is_compatible_with([num_symbols])

with vs.variable_scope(scope or "embedding_rnn_decoder"):

with ops.device("/cpu:0"):

embedding = vs.get_variable("embedding", [num_symbols, cell.input_size])

def extract_argmax_and_embed(prev, _):

"""Loop_function that extracts the symbol from prev and embeds it."""

if output_projection is not None:

prev = nn_ops.xw_plus_b(

prev, output_projection[0], output_projection[1])

prev_symbol = array_ops.stop_gradient(math_ops.argmax(prev, 1))

return embedding_ops.embedding_lookup(embedding, prev_symbol)

loop_function = None

if feed_previous:

loop_function = extract_argmax_and_embed

emb_inp = [

embedding_ops.embedding_lookup(embedding, i) for i in decoder_inputs]

return rnn_decoder(emb_inp, initial_state, cell,

num_encoder_symbols, num_decoder_symbols,

output_projection=None, feed_previous=False,

dtype=dtypes.float32, scope=None):

"""Embedding RNN sequence-to-sequence model.

with vs.variable_scope(scope or "embedding_rnn_seq2seq"):

# Encoder.

encoder_cell = rnn_cell.EmbeddingWrapper(cell, num_encoder_symbols)

_, encoder_states = rnn.rnn(encoder_cell, encoder_inputs, dtype=dtype)

# Decoder.

if output_projection is None:

cell = rnn_cell.OutputProjectionWrapper(cell, num_decoder_symbols)

if isinstance(feed_previous, bool):

return embedding_rnn_decoder(decoder_inputs, encoder_states[-1], cell,

num_decoder_symbols, output_projection,

feed_previous)

else: # If feed_previous is a Tensor, we construct 2 graphs and use cond.

outputs1, states1 = embedding_rnn_decoder(

decoder_inputs, encoder_states[-1], cell, num_decoder_symbols,

output_projection, True)

vs.get_variable_scope().reuse_variables()

outputs2, states2 = embedding_rnn_decoder(

decoder_inputs, encoder_states[-1], cell, num_decoder_symbols,

output_projection, False)

outputs = control_flow_ops.cond(feed_previous,

lambda: outputs1, lambda: outputs2)

states = control_flow_ops.cond(feed_previous,

lambda: states1, lambda: states2)

num_symbols, output_projection=None,

feed_previous=False, dtype=dtypes.float32,

scope=None):

"""Embedding RNN sequence-to-sequence model with tied (shared) parameters.

if output_projection is not None:

proj_weights = ops.convert_to_tensor(output_projection[0], dtype=dtype)

proj_weights.get_shape().assert_is_compatible_with([cell.output_size,

num_symbols])

proj_biases = ops.convert_to_tensor(output_projection[1], dtype=dtype)

proj_biases.get_shape().assert_is_compatible_with([num_symbols])

with vs.variable_scope(scope or "embedding_tied_rnn_seq2seq"):

with ops.device("/cpu:0"):

embedding = vs.get_variable("embedding", [num_symbols, cell.input_size])

emb_encoder_inputs = [embedding_ops.embedding_lookup(embedding, x)

for x in encoder_inputs]

emb_decoder_inputs = [embedding_ops.embedding_lookup(embedding, x)

for x in decoder_inputs]

def extract_argmax_and_embed(prev, _):

"""Loop_function that extracts the symbol from prev and embeds it."""

if output_projection is not None:

prev = nn_ops.xw_plus_b(

prev, output_projection[0], output_projection[1])

prev_symbol = array_ops.stop_gradient(math_ops.argmax(prev, 1))

return embedding_ops.embedding_lookup(embedding, prev_symbol)

if output_projection is None:

cell = rnn_cell.OutputProjectionWrapper(cell, num_symbols)

if isinstance(feed_previous, bool):

loop_function = extract_argmax_and_embed if feed_previous else None

return tied_rnn_seq2seq(emb_encoder_inputs, emb_decoder_inputs, cell,

loop_function=loop_function, dtype=dtype)

else: # If feed_previous is a Tensor, we construct 2 graphs and use cond.

outputs1, states1 = tied_rnn_seq2seq(

emb_encoder_inputs, emb_decoder_inputs, cell,

loop_function=extract_argmax_and_embed, dtype=dtype)

vs.get_variable_scope().reuse_variables()

outputs2, states2 = tied_rnn_seq2seq(

emb_encoder_inputs, emb_decoder_inputs, cell, dtype=dtype)

outputs = control_flow_ops.cond(feed_previous,

lambda: outputs1, lambda: outputs2)

states = control_flow_ops.cond(feed_previous,

lambda: states1, lambda: states2)

output_size=None, num_heads=1, loop_function=None,

dtype=dtypes.float32, scope=None,

initial_state_attention=False):

"""RNN decoder with attention for the sequence-to-sequence model.

if not decoder_inputs:

raise ValueError("Must provide at least 1 input to attention decoder.")

if num_heads < 1:

raise ValueError("With less than 1 heads, use a non-attention decoder.")

if not attention_states.get_shape()[1:2].is_fully_defined():

raise ValueError("Shape[1] and [2] of attention_states must be known: %s"

% attention_states.get_shape())

if output_size is None:

output_size = cell.output_size

with vs.variable_scope(scope or "attention_decoder"):

batch_size = array_ops.shape(decoder_inputs[0])[0] # Needed for reshaping.

attn_length = attention_states.get_shape()[1].value

attn_size = attention_states.get_shape()[2].value

# To calculate W1 * h_t we use a 1-by-1 convolution, need to reshape before.

hidden = array_ops.reshape(

attention_states, [-1, attn_length, 1, attn_size])

hidden_features = []

v = []

attention_vec_size = attn_size # Size of query vectors for attention.

for a in xrange(num_heads):

k = vs.get_variable("AttnW_%d" % a, [1, 1, attn_size, attention_vec_size])

hidden_features.append(nn_ops.conv2d(hidden, k, [1, 1, 1, 1], "SAME"))

v.append(vs.get_variable("AttnV_%d" % a, [attention_vec_size]))

states = [initial_state]

def attention(query):

"""Put attention masks on hidden using hidden_features and query."""

ds = [] # Results of attention reads will be stored here.

for a in xrange(num_heads):

with vs.variable_scope("Attention_%d" % a):

y = rnn_cell.linear(query, attention_vec_size, True)

y = array_ops.reshape(y, [-1, 1, 1, attention_vec_size])

# Attention mask is a softmax of v^T * tanh(...).

s = math_ops.reduce_sum(

v[a] * math_ops.tanh(hidden_features[a] + y), [2, 3])

a = nn_ops.softmax(s)

# Now calculate the attention-weighted vector d.

d = math_ops.reduce_sum(

array_ops.reshape(a, [-1, attn_length, 1, 1]) * hidden,

[1, 2])

ds.append(array_ops.reshape(d, [-1, attn_size]))

return ds

outputs = []

prev = None

batch_attn_size = array_ops.pack([batch_size, attn_size])

attns = [array_ops.zeros(batch_attn_size, dtype=dtype)

for _ in xrange(num_heads)]

for a in attns: # Ensure the second shape of attention vectors is set.

a.set_shape([None, attn_size])

if initial_state_attention:

attns = attention(initial_state)

for i in xrange(len(decoder_inputs)):

if i > 0:

vs.get_variable_scope().reuse_variables()

inp = decoder_inputs[i]

# If loop_function is set, we use it instead of decoder_inputs.

if loop_function is not None and prev is not None:

with vs.variable_scope("loop_function", reuse=True):

inp = array_ops.stop_gradient(loop_function(prev, i))

# Merge input and previous attentions into one vector of the right size.

x = rnn_cell.linear([inp] + attns, cell.input_size, True)

# Run the RNN.

cell_output, new_state = cell(x, states[-1])

states.append(new_state)

# Run the attention mechanism.

if i == 0 and initial_state_attention:

with vs.variable_scope(vs.get_variable_scope(), reuse=True):

attns = attention(new_state)

else:

attns = attention(new_state)

with vs.variable_scope("AttnOutputProjection"):

output = rnn_cell.linear([cell_output] + attns, output_size, True)

if loop_function is not None:

# We do not propagate gradients over the loop function.

prev = array_ops.stop_gradient(output)

outputs.append(output)

cell, num_symbols, num_heads=1,

output_size=None, output_projection=None,

feed_previous=False, dtype=dtypes.float32,

scope=None, initial_state_attention=False):

"""RNN decoder with embedding and attention and a pure-decoding option.

if output_size is None:

output_size = cell.output_size

if output_projection is not None:

proj_weights = ops.convert_to_tensor(output_projection[0], dtype=dtype)

proj_weights.get_shape().assert_is_compatible_with([cell.output_size,

num_symbols])

proj_biases = ops.convert_to_tensor(output_projection[1], dtype=dtype)

proj_biases.get_shape().assert_is_compatible_with([num_symbols])

with vs.variable_scope(scope or "embedding_attention_decoder"):

with ops.device("/cpu:0"):

embedding = vs.get_variable("embedding", [num_symbols, cell.input_size])

def extract_argmax_and_embed(prev, _):

"""Loop_function that extracts the symbol from prev and embeds it."""

if output_projection is not None:

prev = nn_ops.xw_plus_b(

prev, output_projection[0], output_projection[1])

prev_symbol = array_ops.stop_gradient(math_ops.argmax(prev, 1))

emb_prev = embedding_ops.embedding_lookup(embedding, prev_symbol)

return emb_prev

loop_function = None

if feed_previous:

loop_function = extract_argmax_and_embed

emb_inp = [

embedding_ops.embedding_lookup(embedding, i) for i in decoder_inputs]

return attention_decoder(

emb_inp, initial_state, attention_states, cell, output_size=output_size,

num_heads=num_heads, loop_function=loop_function,

num_encoder_symbols, num_decoder_symbols,

num_heads=1, output_projection=None,

feed_previous=False, dtype=dtypes.float32,

scope=None, initial_state_attention=False):

"""Embedding sequence-to-sequence model with attention.

with vs.variable_scope(scope or "embedding_attention_seq2seq"):

# Encoder.

encoder_cell = rnn_cell.EmbeddingWrapper(cell, num_encoder_symbols)

encoder_outputs, encoder_states = rnn.rnn(

encoder_cell, encoder_inputs, dtype=dtype)

# First calculate a concatenation of encoder outputs to put attention on.

top_states = [array_ops.reshape(e, [-1, 1, cell.output_size])

for e in encoder_outputs]

attention_states = array_ops.concat(1, top_states)

# Decoder.

output_size = None

if output_projection is None:

cell = rnn_cell.OutputProjectionWrapper(cell, num_decoder_symbols)

output_size = num_decoder_symbols

if isinstance(feed_previous, bool):

return embedding_attention_decoder(

decoder_inputs, encoder_states[-1], attention_states, cell,

num_decoder_symbols, num_heads, output_size, output_projection,

feed_previous, initial_state_attention=initial_state_attention)

else: # If feed_previous is a Tensor, we construct 2 graphs and use cond.

outputs1, states1 = embedding_attention_decoder(

decoder_inputs, encoder_states[-1], attention_states, cell,

num_decoder_symbols, num_heads, output_size, output_projection, True,

initial_state_attention=initial_state_attention)

vs.get_variable_scope().reuse_variables()

outputs2, states2 = embedding_attention_decoder(

decoder_inputs, encoder_states[-1], attention_states, cell,

num_decoder_symbols, num_heads, output_size, output_projection, False,

initial_state_attention=initial_state_attention)

outputs = control_flow_ops.cond(feed_previous,

lambda: outputs1, lambda: outputs2)

states = control_flow_ops.cond(feed_previous,

lambda: states1, lambda: states2)

average_across_timesteps=True,

softmax_loss_function=None, name=None):

"""Weighted cross-entropy loss for a sequence of logits (per example).

if len(targets) != len(logits) or len(weights) != len(logits):

raise ValueError("Lengths of logits, weights, and targets must be the same "

"%d, %d, %d." % (len(logits), len(weights), len(targets)))

with ops.op_scope(logits + targets + weights, name,

"sequence_loss_by_example"):

batch_size = array_ops.shape(targets[0])[0]

log_perp_list = []

length = batch_size * num_decoder_symbols

for i in xrange(len(logits)):

if softmax_loss_function is None:

# TODO(lukaszkaiser): There is no SparseCrossEntropy in TensorFlow, so

# we need to first cast targets into a dense representation, and as

# SparseToDense does not accept batched inputs, we need to do this by

# re-indexing and re-sizing. When TensorFlow adds SparseCrossEntropy,

# rewrite this method.

indices = targets[i] + num_decoder_symbols * math_ops.range(batch_size)

with ops.device("/cpu:0"): # Sparse-to-dense must be on CPU for now.

dense = sparse_ops.sparse_to_dense(

indices, array_ops.expand_dims(length, 0), 1.0,

0.0)

target = array_ops.reshape(dense, [-1, num_decoder_symbols])

crossent = nn_ops.softmax_cross_entropy_with_logits(

logits[i], target, name="SequenceLoss/CrossEntropy{0}".format(i))

else:

crossent = softmax_loss_function(logits[i], targets[i])

log_perp_list.append(crossent * weights[i])

log_perps = math_ops.add_n(log_perp_list)

if average_across_timesteps:

total_size = math_ops.add_n(weights)

total_size += 1e-12 # Just to avoid division by 0 for all-0 weights.

log_perps /= total_size

average_across_timesteps=True, average_across_batch=True,

softmax_loss_function=None, name=None):

"""Weighted cross-entropy loss for a sequence of logits, batch-collapsed.

with ops.op_scope(logits + targets + weights, name, "sequence_loss"):

cost = math_ops.reduce_sum(sequence_loss_by_example(

logits, targets, weights, num_decoder_symbols,

average_across_timesteps=average_across_timesteps,

softmax_loss_function=softmax_loss_function))

if average_across_batch:

batch_size = array_ops.shape(targets[0])[0]

return cost / math_ops.cast(batch_size, dtypes.float32)

else:

buckets, num_decoder_symbols, seq2seq,

softmax_loss_function=None, name=None):

"""Create a sequence-to-sequence model with support for bucketing.

if len(encoder_inputs) < buckets[-1][0]:

raise ValueError("Length of encoder_inputs (%d) must be at least that of la"

"st bucket (%d)." % (len(encoder_inputs), buckets[-1][0]))

if len(targets) < buckets[-1][1]:

raise ValueError("Length of targets (%d) must be at least that of last"

"bucket (%d)." % (len(targets), buckets[-1][1]))

if len(weights) < buckets[-1][1]:

raise ValueError("Length of weights (%d) must be at least that of last"

"bucket (%d)." % (len(weights), buckets[-1][1]))

all_inputs = encoder_inputs + decoder_inputs + targets + weights

losses = []

outputs = []

with ops.op_scope(all_inputs, name, "model_with_buckets"):

for j in xrange(len(buckets)):

with vs.variable_scope(vs.get_variable_scope(),

reuse=True if j > 0 else None):

bucket_encoder_inputs = [encoder_inputs[i]

for i in xrange(buckets[j][0])]

bucket_decoder_inputs = [decoder_inputs[i]

for i in xrange(buckets[j][1])]

bucket_outputs, _ = seq2seq(bucket_encoder_inputs,

bucket_decoder_inputs)

outputs.append(bucket_outputs)

bucket_targets = [targets[i] for i in xrange(buckets[j][1])]

bucket_weights = [weights[i] for i in xrange(buckets[j][1])]

losses.append(sequence_loss(

outputs[-1], bucket_targets, bucket_weights, num_decoder_symbols,

softmax_loss_function=softmax_loss_function))