Merge pull request #751 from rsepassi/push

v1.6.1

.travis.yml

@@ -41,6 +41,7 @@ script:
     --ignore=tensor2tensor/problems_test.py
     --ignore=tensor2tensor/bin/t2t_trainer_test.py
     --ignore=tensor2tensor/data_generators/algorithmic_math_test.py
+    --ignore=tensor2tensor/models/research/r_transformer_test.py  # Requires new feature in tf.foldl (rm with TF 1.9)
   - pytest tensor2tensor/utils/registry_test.py
   - pytest tensor2tensor/utils/trainer_lib_test.py
   - pytest tensor2tensor/visualization/visualization_test.py

docs/distributed_training.md

@@ -51,8 +51,10 @@ distributed training:
 Parameter servers only need `--master=grpc://$ADDRESS` and
 `--schedule=run_std_server`.
 
->> Note about `output_dir`: All the workers (masters and parameter servers) should use the same `output_dir`. If training
->> on separate nodes, output_dir can be a shared filesystem like NFS or an object store like GCS.
+>> Note about `--output_dir`: All the nodes should use the same `--output_dir`.
+>> When using multiple machines, `output_dir` should point to a shared
+>> filesystem like NFS or an object store like Google Cloud Storage
+>> (`gs://...`).
 
 ## Utility to produce `TF_CONFIG` and flags
 

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='tensor2tensor',
-    version='1.6.0',
+    version='1.6.1',
     description='Tensor2Tensor',
     author='Google Inc.',
     author_email='[email protected]',
@@ -14,6 +14,7 @@
     packages=find_packages(),
     package_data={
         'tensor2tensor.data_generators': ['test_data/*'],
+        'tensor2tensor.data_generators.wikisum': ['test_data/*'],
         'tensor2tensor.visualization': [
             'attention.js', 'TransformerVisualization.ipynb'
         ],
@@ -37,7 +38,8 @@
         'gevent',
         'google-api-python-client',
         'gunicorn',
-        'gym<=0.9.5',  # gym in version 0.9.6 has some temporary issues.
+        'gym',
+        'h5py',
         'numpy',
         'requests',
         'scipy',
@@ -47,7 +49,7 @@
     extras_require={
         'tensorflow': ['tensorflow>=1.5.0'],
         'tensorflow_gpu': ['tensorflow-gpu>=1.5.0'],
-        'tests': ['pytest', 'h5py', 'mock'],
+        'tests': ['pytest', 'mock'],
     },
     classifiers=[
         'Development Status :: 4 - Beta',

tensor2tensor/bin/make_tf_configs.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Output command line arguments and json-encoded TF_CONFIGs.
 
 Usage:

tensor2tensor/bin/t2t_avg_all.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Script to continuously average last N checkpoints in a given directory."""
 from __future__ import absolute_import
 from __future__ import division

tensor2tensor/bin/t2t_bleu.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Evaluate BLEU score for all checkpoints/translations in a given directory.
 
 This script can be used in two ways.

tensor2tensor/bin/t2t_datagen.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Produces the training and dev data for --problem into --data_dir.
 
 Produces sharded and shuffled TFRecord files of tensorflow.Example protocol

tensor2tensor/bin/t2t_decoder.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 r"""Decode from trained T2T models.
 
 This binary performs inference using the Estimator API.
@@ -82,9 +81,13 @@ def create_decode_hparams():
 
 def decode(estimator, hparams, decode_hp):
   if FLAGS.decode_interactive:
+    if estimator.config.use_tpu:
+      raise ValueError("TPU can only decode from dataset.")
     decoding.decode_interactively(estimator, hparams, decode_hp,
                                   checkpoint_path=FLAGS.checkpoint_path)
   elif FLAGS.decode_from_file:
+    if estimator.config.use_tpu:
+      raise ValueError("TPU can only decode from dataset.")
     decoding.decode_from_file(estimator, FLAGS.decode_from_file, hparams,
                               decode_hp, FLAGS.decode_to_file,
                               checkpoint_path=FLAGS.checkpoint_path)
@@ -160,7 +163,6 @@ def main(_):
   tf.logging.set_verbosity(tf.logging.INFO)
   trainer_lib.set_random_seed(FLAGS.random_seed)
   usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
-  FLAGS.use_tpu = False  # decoding not supported on TPU
 
   if FLAGS.score_file:
     filename = os.path.expanduser(FLAGS.score_file)
@@ -183,7 +185,7 @@ def main(_):
       hp,
       t2t_trainer.create_run_config(hp),
       decode_hparams=decode_hp,
-      use_tpu=False)
+      use_tpu=FLAGS.use_tpu)
 
   decode(estimator, hp, decode_hp)
 

tensor2tensor/bin/t2t_distill.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 r"""Perform distillation for a teacher to student.
 
 This script is intended to be used with --model=distillation. See the model for

tensor2tensor/bin/t2t_trainer.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Train and evaluate."""
 from __future__ import absolute_import
 from __future__ import division
@@ -87,6 +86,8 @@
                     "Name of Cloud TPU instance to use or create.")
 flags.DEFINE_bool("cloud_delete_on_done", False,
                   "Whether to delete the VM and TPU instance when done.")
+flags.DEFINE_bool("cloud_skip_confirmation", False,
+                  "Whether to skip launch confirmations.")
 
 # Google Cloud ML Engine
 flags.DEFINE_bool("cloud_mlengine", False,
@@ -319,7 +320,8 @@ def maybe_cloud_tpu():
   with cloud_tpu.cloud_tpu(
       FLAGS.cloud_vm_name,
       FLAGS.cloud_tpu_name,
-      delete_on_done=FLAGS.cloud_delete_on_done) as tpu_master:
+      delete_on_done=FLAGS.cloud_delete_on_done,
+      skip_confirmation=FLAGS.cloud_skip_confirmation) as tpu_master:
     FLAGS.master = tpu_master
     yield
 

tensor2tensor/bin/t2t_trainer_test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Tests for t2t_trainer."""
 
 from __future__ import absolute_import

tensor2tensor/bin/t2t_translate_all.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Translate a file with all checkpoints in a given directory.
 
 t2t-decoder will be executed with these parameters:

tensor2tensor/data_generators/algorithmic.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Algorithmic data generators."""
 from __future__ import absolute_import
 from __future__ import division

tensor2tensor/data_generators/algorithmic_math.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Algorithmic data generators for symbolic math tasks.
 
 See go/symbolic-math-dataset

tensor2tensor/data_generators/algorithmic_math_test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Tests for tensor2tensor.data_generators.algorithmic_math."""
 # TODO(rsepassi): This test is flaky. Disable, remove, or update.
 

tensor2tensor/data_generators/algorithmic_test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Algorithmic generators test."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/all_problems.py

@@ -12,13 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Imports for problem modules."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
 import importlib
+import six
 
 
 modules = [
@@ -55,12 +55,39 @@
     "tensor2tensor.data_generators.translate_enzh",
     "tensor2tensor.data_generators.twentybn",
     "tensor2tensor.data_generators.wiki",
+    "tensor2tensor.data_generators.wikisum.wikisum",
     "tensor2tensor.data_generators.wsj_parsing",
 ]
 
 
-for module in modules:
+
+def _py_err_msg(module):
+  if six.PY2:
+    msg = "No module named %s" % module.split(".")[-1]
+  else:
+    msg = "No module named '%s'" % module
+  return msg
+
+
+def _handle_errors(errors):
+  """Log out and possibly reraise errors during import."""
+  if not errors:
+    return
+  log_all = True  # pylint: disable=unused-variable
+  err_msg = "Skipped importing {num_missing} data_generators modules."
+  print(err_msg.format(num_missing=len(errors)))
+  for module, err in errors:
+    err_str = str(err)
+    if err_str != _py_err_msg(module):
+      raise err
+    if log_all:
+      print("Did not import module: %s; Cause: %s" % (module, err_str))
+
+
+_errors = []
+for _module in modules:
   try:
-    importlib.import_module(module)
+    importlib.import_module(_module)
   except ImportError as error:
-    print("Did not import module: %s; Cause: %s" % (module, str(error)))
+    _errors.append((_module, error))
+_handle_errors(_errors)

tensor2tensor/data_generators/all_problems_test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Tests for Tensor2Tensor's all_problems.py."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/audio.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """TIMIT data generator."""
 from __future__ import absolute_import
 from __future__ import division

tensor2tensor/data_generators/audio_test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Tests for tensor2tensor.data_generators.audio."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/celeba.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """CelebA."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/cifar.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """CIFAR."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/cipher.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Cipher data generators."""
 from __future__ import absolute_import
 from __future__ import division

tensor2tensor/data_generators/cnn_dailymail.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Data generators for the CNN and Daily Mail datasets."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/desc2code.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Data generators for the Description2Code OpenAI data-set."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/desc2code_test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Tests for desc2code."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/dna_encoder.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Encoders for DNA data.
 
 * DNAEncoder: ACTG strings to ints and back

tensor2tensor/data_generators/dna_encoder_test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Tests for tensor2tensor.data_generators.dna_encoder."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/fsns.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """FSNS."""
 
 from __future__ import absolute_import

tensor2tensor/data_generators/gene_expression.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Gene expression problems.
 
 Inputs are bases ACTG (with indices assigned in that order).

tensor2tensor/data_generators/gene_expression_test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Tests for Genetics problems."""
 from __future__ import absolute_import
 from __future__ import division