diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 7bc23fe8..dd47acb1 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -2,13 +2,10 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="69f12537-6c3d-43b1-a62a-ad07d70a14cd" name="Default" comment="">
-      <change type="DELETED" beforePath="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/cf_gan.py" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/dis_model.py" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/gen_model.py" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/utils.py" afterPath="" />
       <change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
-      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/ml-100k/dis-train.txt" afterPath="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/ml-100k/dis-train.txt" />
-      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/ml-100k/gen_log.txt" afterPath="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/ml-100k/gen_log.txt" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/CV/ResNet/resnet_mnist.py" afterPath="$PROJECT_DIR$/CV/ResNet/resnet_mnist.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/recommendation/Basic-DIN-Demo/model.py" afterPath="$PROJECT_DIR$/recommendation/Basic-DIN-Demo/model.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/recommendation/recommendation-FM-demo/FM_model.py" afterPath="$PROJECT_DIR$/recommendation/recommendation-FM-demo/FM_model.py" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="TRACKING_ENABLED" value="true" />
@@ -19,6 +16,7 @@
   </component>
   <component name="CoverageDataManager">
     <SUITE FILE_PATH="coverage/tensorflow1_2$netease.coverage" NAME="netease Coverage Results" MODIFIED="1534173544289" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation" />
+    <SUITE FILE_PATH="coverage/tensorflow1_2$resnet_mnist.coverage" NAME="resnet_mnist Coverage Results" MODIFIED="1540611064870" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/CV/ResNet" />
     <SUITE FILE_PATH="coverage/tensorflow1_2$gen_model.coverage" NAME="gen_model Coverage Results" MODIFIED="1532186281780" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo" />
     <SUITE FILE_PATH="coverage/tensorflow1_2$zhihu.coverage" NAME="zhihu Coverage Results" MODIFIED="1535904633696" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
     <SUITE FILE_PATH="coverage/tensorflow1_2$lr.coverage" NAME="lr Coverage Results" MODIFIED="1529722387251" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation/Basic-MLR-Demo" />
@@ -34,119 +32,114 @@
     <SUITE FILE_PATH="coverage/tensorflow1_2$cf_gan.coverage" NAME="cf_gan Coverage Results" MODIFIED="1537765219138" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo" />
     <SUITE FILE_PATH="coverage/tensorflow1_2$plotResult.coverage" NAME="plotResult Coverage Results" MODIFIED="1529723400750" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation/Basic-MLR-Demo" />
     <SUITE FILE_PATH="coverage/tensorflow1_2$tf_practice.coverage" NAME="tf_practice Coverage Results" MODIFIED="1537593989869" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/test_some_code" />
-    <SUITE FILE_PATH="coverage/tensorflow1_2$train.coverage" NAME="train Coverage Results" MODIFIED="1537519892338" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo" />
+    <SUITE FILE_PATH="coverage/tensorflow1_2$train.coverage" NAME="train Coverage Results" MODIFIED="1541326738678" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo" />
     <SUITE FILE_PATH="coverage/tensorflow1_2$DRQN.coverage" NAME="DRQN Coverage Results" MODIFIED="1530434931296" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/RL/Basic-DRQN-Demo" />
     <SUITE FILE_PATH="coverage/tensorflow1_2$data.coverage" NAME="data Coverage Results" MODIFIED="1529689500332" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/recommendation/Basic-MLR-Demo" />
   </component>
   <component name="FileEditorManager">
     <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
-      <file leaf-file-name="tf_practice.py" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/test_some_code/tf_practice.py">
+      <file leaf-file-name="train.py" pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/train.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="0">
-              <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="18" selection-end-column="0" />
+            <state relative-caret-position="-88">
+              <caret line="115" column="142" lean-forward="false" selection-start-line="115" selection-start-column="142" selection-end-line="115" selection-end-column="142" />
               <folding>
-                <element signature="e#0#23#0" expanded="true" />
+                <element signature="e#0#18#0" expanded="true" />
               </folding>
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="infer.userid.txt" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/xdeepfm-data/dnn/infer.userid.txt">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-615">
-              <caret line="0" column="2" lean-forward="true" selection-start-line="0" selection-start-column="2" selection-end-line="0" selection-end-column="10" />
-              <folding />
+      <file leaf-file-name="local_train_splitByUser" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/data/local_train_splitByUser">
+          <provider selected="true" editor-type-id="LargeFileEditor">
+            <state relative-caret-position="0">
+              <caret line="0" column="8" lean-forward="false" selection-start-line="0" selection-start-column="8" selection-end-line="0" selection-end-column="8" />
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="exDeepFM.py" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/src/exDeepFM.py">
+      <file leaf-file-name="model.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/model.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-630">
-              <caret line="209" column="40" lean-forward="true" selection-start-line="209" selection-start-column="40" selection-end-line="209" selection-end-column="40" />
-              <folding />
+            <state relative-caret-position="346">
+              <caret line="88" column="17" lean-forward="false" selection-start-line="88" selection-start-column="17" selection-end-line="88" selection-end-column="17" />
+              <folding>
+                <element signature="e#0#18#0" expanded="true" />
+              </folding>
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="cf_gan.py" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/cf_gan.py">
+      <file leaf-file-name="data_iterator.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/data_iterator.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="350">
-              <caret line="199" column="5" lean-forward="true" selection-start-line="199" selection-start-column="5" selection-end-line="199" selection-end-column="5" />
+            <state relative-caret-position="157">
+              <caret line="137" column="23" lean-forward="false" selection-start-line="137" selection-start-column="23" selection-end-line="137" selection-end-column="23" />
               <folding>
-                <element signature="e#0#23#0" expanded="true" />
+                <element signature="e#0#18#0" expanded="true" />
               </folding>
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="GAN.py" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/GAN/GAN.py">
+      <file leaf-file-name="rnn.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/rnn.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="180">
-              <caret line="23" column="4" lean-forward="false" selection-start-line="23" selection-start-column="4" selection-end-line="23" selection-end-column="4" />
+            <state relative-caret-position="161">
+              <caret line="552" column="0" lean-forward="false" selection-start-line="552" selection-start-column="0" selection-end-line="552" selection-end-column="0" />
               <folding>
-                <element signature="e#0#23#0" expanded="true" />
+                <element signature="e#852#890#0" expanded="true" />
               </folding>
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="RL_brain.py" pinned="false" current-in-tab="true">
-        <entry file="file://$PROJECT_DIR$/RL/Basic-Policy-Network/RL_brain.py">
+      <file leaf-file-name="rnn_cell_impl.py" pinned="false" current-in-tab="false">
+        <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-744">
-              <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+            <state relative-caret-position="5">
+              <caret line="194" column="71" lean-forward="false" selection-start-line="194" selection-start-column="71" selection-end-line="194" selection-end-column="71" />
               <folding />
             </state>
           </provider>
         </entry>
       </file>
       <file leaf-file-name="utils.py" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/utils.py">
+        <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/utils.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="1575">
-              <caret line="105" column="0" lean-forward="true" selection-start-line="105" selection-start-column="0" selection-end-line="105" selection-end-column="0" />
-              <folding>
-                <element signature="e#0#16#0" expanded="true" />
-              </folding>
+            <state relative-caret-position="561">
+              <caret line="58" column="0" lean-forward="false" selection-start-line="58" selection-start-column="0" selection-end-line="58" selection-end-column="0" />
+              <folding />
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="dis_model.py" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/dis_model.py">
+      <file leaf-file-name="vecAttGruCell.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/vecAttGruCell.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="437">
-              <caret line="60" column="0" lean-forward="false" selection-start-line="60" selection-start-column="0" selection-end-line="60" selection-end-column="0" />
-              <folding>
-                <element signature="e#0#23#0" expanded="true" />
-              </folding>
+            <state relative-caret-position="90">
+              <caret line="6" column="43" lean-forward="false" selection-start-line="6" selection-start-column="43" selection-end-line="6" selection-end-column="43" />
+              <folding />
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="gen_model.py" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/gen_model.py">
+      <file leaf-file-name="decorators.py" pinned="false" current-in-tab="false">
+        <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/nltk/decorators.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="452">
-              <caret line="54" column="0" lean-forward="false" selection-start-line="54" selection-start-column="0" selection-end-line="54" selection-end-column="0" />
-              <folding>
-                <element signature="e#0#23#0" expanded="true" />
-              </folding>
+            <state relative-caret-position="223">
+              <caret line="131" column="8" lean-forward="false" selection-start-line="131" selection-start-column="8" selection-end-line="131" selection-end-column="8" />
+              <folding />
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="main.py" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/main.py">
+      <file leaf-file-name="Dice.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/Dice.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="655">
-              <caret line="143" column="29" lean-forward="false" selection-start-line="143" selection-start-column="29" selection-end-line="143" selection-end-column="29" />
+            <state relative-caret-position="345">
+              <caret line="23" column="0" lean-forward="false" selection-start-line="23" selection-start-column="0" selection-end-line="23" selection-end-column="0" />
               <folding />
             </state>
           </provider>
@@ -171,6 +164,8 @@
       <find>next_hidden</find>
       <find>hidden_nn_layers</find>
       <find>save_model</find>
+      <find>&quot;Abstract method&quot;</find>
+      <find>print</find>
     </findStrings>
   </component>
   <component name="Git.Settings">
@@ -180,20 +175,6 @@
   <component name="IdeDocumentHistory">
     <option name="CHANGED_PATHS">
       <list>
-        <option value="$PROJECT_DIR$/recommendation/Basic-NCF-Demo/metrics.py" />
-        <option value="$PROJECT_DIR$/recommendation/Basic-NCF-Demo/main.py" />
-        <option value="$PROJECT_DIR$/recommendation/Basic-NCF-Demo/tf_data_demo.py" />
-        <option value="$PROJECT_DIR$/recommendation/Basic-Bandit-Demo/Baisc-UCB.py" />
-        <option value="$PROJECT_DIR$/recommendation/Basic-Bandit-Demo/Basic-LinUCB.py" />
-        <option value="$PROJECT_DIR$/RL/Basic-MADDPG-Demo/model_agent_maddpg.py" />
-        <option value="$PROJECT_DIR$/RL/Basic-MADDPG-Demo/three_agent_maddpg.py" />
-        <option value="$PROJECT_DIR$/RL/Basic-Evaluation-metrics/AUC.py" />
-        <option value="$PROJECT_DIR$/RL/Basic-Evaluation-metrics/HR.py" />
-        <option value="$PROJECT_DIR$/RL/Basic-Evaluation-metrics/NDCG.py" />
-        <option value="$PROJECT_DIR$/RL/Basic-Evaluation-metrics/MRR.py" />
-        <option value="$PROJECT_DIR$/RL/Basic-Evaluation-metrics/MAE.py" />
-        <option value="$PROJECT_DIR$/RL/Basic-Evaluation-metrics/MAP.py" />
-        <option value="$PROJECT_DIR$/recommendation/Basic-Evaluation-metrics/NDCG.py" />
         <option value="$PROJECT_DIR$/recommendation/Basic-Evaluation-metrics/HR.py" />
         <option value="$PROJECT_DIR$/recommendation/Basic-Evaluation-metrics/MRR.py" />
         <option value="$PROJECT_DIR$/recommendation/Basic-NCF-Demo/NCF_input.py" />
@@ -231,6 +212,20 @@
         <option value="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/gen_model.py" />
         <option value="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/dis_model.py" />
         <option value="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/cf_gan.py" />
+        <option value="$PROJECT_DIR$/recommendation/recommendation-FM-demo/FM_model.py" />
+        <option value="$PROJECT_DIR$/CV/CNN.py" />
+        <option value="$PROJECT_DIR$/CV/ResNet/resnet_mnist.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/shuffle.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIN-Demo/model.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/utils.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/VecAttGruCell.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/rnn.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/model.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/vecAttGruCell.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/Dice.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/data_iterator.py" />
+        <option value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/train.py" />
       </list>
     </option>
   </component>
@@ -259,7 +254,6 @@
       <foldersAlwaysOnTop value="true" />
     </navigator>
     <panes>
-      <pane id="Scope" />
       <pane id="ProjectPane">
         <subPane>
           <expand>
@@ -267,56 +261,23 @@
               <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
               <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
             </path>
-            <path>
-              <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
-              <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
-              <item name="basic" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
-              <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
-              <item name="GAN" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
-              <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
-              <item name="recommendation" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
-              <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
-              <item name="recommendation" type="462c0819:PsiDirectoryNode" />
-              <item name="Basic-Evaluation-metrics" type="462c0819:PsiDirectoryNode" />
-            </path>
             <path>
               <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
               <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
               <item name="recommendation" type="462c0819:PsiDirectoryNode" />
-              <item name="Basic-IRGAN-Demo" type="462c0819:PsiDirectoryNode" />
             </path>
             <path>
               <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
               <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
               <item name="recommendation" type="462c0819:PsiDirectoryNode" />
-              <item name="Basic-IRGAN-Demo" type="462c0819:PsiDirectoryNode" />
-              <item name="ml-100k" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
-              <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
-              <item name="RL" type="462c0819:PsiDirectoryNode" />
-            </path>
-            <path>
-              <item name="tensorflow1.2" type="b2602c69:ProjectViewProjectNode" />
-              <item name="tensorflow1.2" type="462c0819:PsiDirectoryNode" />
-              <item name="RL" type="462c0819:PsiDirectoryNode" />
-              <item name="Basic-Policy-Network" type="462c0819:PsiDirectoryNode" />
+              <item name="Basic-DIEN-Demo" type="462c0819:PsiDirectoryNode" />
             </path>
           </expand>
           <select />
         </subPane>
       </pane>
       <pane id="Scratches" />
+      <pane id="Scope" />
     </panes>
   </component>
   <component name="PropertiesComponent">
@@ -325,18 +286,19 @@
     <property name="WebServerToolWindowFactoryState" value="false" />
   </component>
   <component name="RecentsManager">
-    <key name="MoveFile.RECENT_KEYS">
-      <recent name="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/test_some_code" />
-      <recent name="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo" />
-      <recent name="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/res" />
-      <recent name="$PROJECT_DIR$/recommendation" />
-    </key>
     <key name="CopyFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo" />
       <recent name="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/ml-100k" />
       <recent name="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/test_some_code" />
       <recent name="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo" />
       <recent name="$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo" />
-      <recent name="$PROJECT_DIR$/recommendation" />
+    </key>
+    <key name="MoveFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/data" />
+      <recent name="$PROJECT_DIR$/CV" />
+      <recent name="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/test_some_code" />
+      <recent name="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo" />
+      <recent name="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/res" />
     </key>
   </component>
   <component name="RunDashboard">
@@ -351,7 +313,7 @@
       </list>
     </option>
   </component>
-  <component name="RunManager" selected="Python.cf_gan">
+  <component name="RunManager" selected="Python.train">
     <configuration name="cf_gan" type="PythonConfigurationType" factoryName="Python" temporary="true">
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
@@ -370,38 +332,38 @@
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
     </configuration>
-    <configuration name="main" type="PythonConfigurationType" factoryName="Python" temporary="true">
+    <configuration name="maxout" type="PythonConfigurationType" factoryName="Python" temporary="true">
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
       <envs>
         <env name="PYTHONUNBUFFERED" value="1" />
       </envs>
       <option name="SDK_HOME" value="" />
-      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basic" />
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
       <module name="tensorflow1.2" />
       <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/main.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/basic/maxout.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
     </configuration>
-    <configuration name="maxout" type="PythonConfigurationType" factoryName="Python" temporary="true">
+    <configuration name="resnet_mnist" type="PythonConfigurationType" factoryName="Python" temporary="true">
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
       <envs>
         <env name="PYTHONUNBUFFERED" value="1" />
       </envs>
       <option name="SDK_HOME" value="" />
-      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basic" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/CV/ResNet" />
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
       <module name="tensorflow1.2" />
       <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/basic/maxout.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/CV/ResNet/resnet_mnist.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
@@ -431,31 +393,31 @@
         <env name="PYTHONUNBUFFERED" value="1" />
       </envs>
       <option name="SDK_HOME" value="" />
-      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo" />
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
       <module name="tensorflow1.2" />
       <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/train.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/train.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
     </configuration>
     <list size="5">
-      <item index="0" class="java.lang.String" itemvalue="Python.train" />
-      <item index="1" class="java.lang.String" itemvalue="Python.main" />
-      <item index="2" class="java.lang.String" itemvalue="Python.tf_practice" />
-      <item index="3" class="java.lang.String" itemvalue="Python.maxout" />
-      <item index="4" class="java.lang.String" itemvalue="Python.cf_gan" />
+      <item index="0" class="java.lang.String" itemvalue="Python.tf_practice" />
+      <item index="1" class="java.lang.String" itemvalue="Python.maxout" />
+      <item index="2" class="java.lang.String" itemvalue="Python.cf_gan" />
+      <item index="3" class="java.lang.String" itemvalue="Python.resnet_mnist" />
+      <item index="4" class="java.lang.String" itemvalue="Python.train" />
     </list>
     <recent_temporary>
       <list size="5">
-        <item index="0" class="java.lang.String" itemvalue="Python.cf_gan" />
-        <item index="1" class="java.lang.String" itemvalue="Python.maxout" />
-        <item index="2" class="java.lang.String" itemvalue="Python.tf_practice" />
-        <item index="3" class="java.lang.String" itemvalue="Python.main" />
-        <item index="4" class="java.lang.String" itemvalue="Python.train" />
+        <item index="0" class="java.lang.String" itemvalue="Python.train" />
+        <item index="1" class="java.lang.String" itemvalue="Python.resnet_mnist" />
+        <item index="2" class="java.lang.String" itemvalue="Python.cf_gan" />
+        <item index="3" class="java.lang.String" itemvalue="Python.maxout" />
+        <item index="4" class="java.lang.String" itemvalue="Python.tf_practice" />
       </list>
     </recent_temporary>
   </component>
@@ -483,16 +445,17 @@
   </component>
   <component name="ToolWindowManager">
     <frame x="0" y="0" width="1440" height="900" extended-state="0" />
+    <editor active="true" />
     <layout>
-      <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.36194563" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
+      <window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.19027181" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
       <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
       <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
       <window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
       <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32889965" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
       <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
-      <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.28778717" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
+      <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.5090689" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
       <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
-      <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.27811366" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.2756953" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
       <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
       <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.39782345" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
       <window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
@@ -533,340 +496,376 @@
           </properties>
         </breakpoint>
       </default-breakpoints>
-      <option name="time" value="6" />
+      <option name="time" value="7" />
     </breakpoint-manager>
     <watches-manager />
   </component>
   <component name="editorHistoryManager">
-    <entry file="file://$PROJECT_DIR$/GAN/seqgan/generator.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/utils/metric.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="525">
-          <caret line="120" column="30" lean-forward="false" selection-start-line="120" selection-start-column="30" selection-end-line="120" selection-end-column="30" />
+        <state relative-caret-position="490">
+          <caret line="97" column="0" lean-forward="false" selection-start-line="97" selection-start-column="0" selection-end-line="97" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/train.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="136">
+          <caret line="168" column="0" lean-forward="false" selection-start-line="168" selection-start-column="0" selection-end-line="168" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/ml-100k/gan_generator.pkl">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/IO/ffm_cache.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="31">
+          <caret line="67" column="29" lean-forward="false" selection-start-line="67" selection-start-column="29" selection-end-line="67" selection-end-column="29" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/IO/base_cache.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="195">
+          <caret line="13" column="64" lean-forward="false" selection-start-line="13" selection-start-column="64" selection-end-line="13" selection-end-column="64" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/IO/iterator.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="2224">
+          <caret line="207" column="0" lean-forward="false" selection-start-line="207" selection-start-column="0" selection-end-line="207" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/src/base_model.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="2745">
+          <caret line="183" column="78" lean-forward="false" selection-start-line="183" selection-start-column="78" selection-end-line="183" selection-end-column="78" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/cache/eval_num.csv" />
+    <entry file="file://$PROJECT_DIR$/recommendation/Newton Interpolation.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="0">
           <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/GBDT+LR-Demo/data/train.csv">
+    <entry file="file://$PROJECT_DIR$/recommendation/multi-value-feature.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-16">
+        <state relative-caret-position="0">
           <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/utils/util.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="337">
-          <caret line="702" column="69" lean-forward="true" selection-start-line="702" selection-start-column="69" selection-end-line="702" selection-end-column="69" />
+        <state relative-caret-position="270">
+          <caret line="21" column="0" lean-forward="false" selection-start-line="21" selection-start-column="0" selection-end-line="21" selection-end-column="14" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/variables.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/config/network.yaml">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-817">
-          <caret line="122" column="6" lean-forward="false" selection-start-line="122" selection-start-column="6" selection-end-line="122" selection-end-column="6" />
+        <state relative-caret-position="90">
+          <caret line="6" column="28" lean-forward="false" selection-start-line="6" selection-start-column="28" selection-end-line="6" selection-end-column="28" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/basic/multi-task-Demo/multi-task-alternate-training.py">
+    <entry file="file://$PROJECT_DIR$/.gitignore">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="255">
-          <caret line="19" column="32" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="39" selection-end-column="31" />
+        <state relative-caret-position="285">
+          <caret line="19" column="11" lean-forward="false" selection-start-line="19" selection-start-column="11" selection-end-line="19" selection-end-column="11" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/basic/CNN.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/test_some_code/tf_practice.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="0">
-          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="18" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$USER_HOME$/Library/Caches/PyCharm2017.2/python_stubs/-1877576585/numpy/random/mtrand.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/main.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="29475">
-          <caret line="1969" column="4" lean-forward="false" selection-start-line="1969" selection-start-column="4" selection-end-line="1969" selection-end-column="4" />
+        <state relative-caret-position="2085">
+          <caret line="143" column="29" lean-forward="false" selection-start-line="143" selection-start-column="29" selection-end-line="143" selection-end-column="29" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/xdeepfm-data/dnn/infer.userid.txt">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="2" lean-forward="false" selection-start-line="0" selection-start-column="2" selection-end-line="0" selection-end-column="10" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/basic/multi-task-Demo/multi-task-joint-training.py">
+    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="315">
-          <caret line="23" column="36" lean-forward="false" selection-start-line="23" selection-start-column="36" selection-end-line="23" selection-end-column="36" />
+        <state relative-caret-position="-295">
+          <caret line="2370" column="4" lean-forward="false" selection-start-line="2370" selection-start-column="4" selection-end-line="2370" selection-end-column="4" />
+          <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/netease.py" />
-    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/codecs.py">
+    <entry file="file://$PROJECT_DIR$/basic/maxout.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="4620">
-          <caret line="308" column="0" lean-forward="false" selection-start-line="308" selection-start-column="0" selection-end-line="308" selection-end-column="0" />
+        <state relative-caret-position="165">
+          <caret line="11" column="15" lean-forward="false" selection-start-line="11" selection-start-column="15" selection-end-line="11" selection-end-column="15" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/zhihu2.py" />
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DeepFM-model/DeepFM.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/src/exDeepFM.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="479">
-          <caret line="103" column="57" lean-forward="false" selection-start-line="103" selection-start-column="35" selection-end-line="103" selection-end-column="57" />
+        <state relative-caret-position="3090">
+          <caret line="209" column="40" lean-forward="false" selection-start-line="209" selection-start-column="40" selection-end-line="209" selection-end-column="40" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/utils/log.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/ml-100k/model_dns_ori.pkl">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="330">
-          <caret line="22" column="0" lean-forward="true" selection-start-line="22" selection-start-column="0" selection-end-line="22" selection-end-column="0" />
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/data/dnn/val.userid.txt" />
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/cache/batch_size_128_infer.userid.tfrecord" />
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/cache/feat_cnt.csv" />
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/cache/infer_num.csv" />
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/res/exDeepFM.py" />
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/cache/infer_impressionId.csv" />
-    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/sparse_tensor.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/utils.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="1440">
-          <caret line="104" column="6" lean-forward="false" selection-start-line="104" selection-start-column="6" selection-end-line="104" selection-end-column="6" />
-          <folding />
+        <state relative-caret-position="1575">
+          <caret line="105" column="0" lean-forward="false" selection-start-line="105" selection-start-column="0" selection-end-line="105" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/data/dnn/infer.userid.txt" />
-    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py">
+    <entry file="file://$PROJECT_DIR$/GAN/GAN.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="267">
-          <caret line="1213" column="4" lean-forward="false" selection-start-line="1213" selection-start-column="4" selection-end-line="1213" selection-end-column="4" />
-          <folding />
+        <state relative-caret-position="345">
+          <caret line="23" column="4" lean-forward="false" selection-start-line="23" selection-start-column="4" selection-end-line="23" selection-end-column="4" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/zhihu.py" />
-    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/random_ops.py">
+    <entry file="file://$PROJECT_DIR$/RL/Basic-Policy-Network/RL_brain.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="2921">
-          <caret line="318" column="4" lean-forward="false" selection-start-line="318" selection-start-column="4" selection-end-line="318" selection-end-column="4" />
-          <folding />
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/utils/metric.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/gen_model.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="490">
-          <caret line="97" column="0" lean-forward="false" selection-start-line="97" selection-start-column="0" selection-end-line="97" selection-end-column="0" />
-          <folding />
+        <state relative-caret-position="765">
+          <caret line="51" column="0" lean-forward="false" selection-start-line="40" selection-start-column="8" selection-end-line="51" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIN-Demo/model.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/dis_model.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="143">
-          <caret line="136" column="8" lean-forward="false" selection-start-line="136" selection-start-column="8" selection-end-line="136" selection-end-column="8" />
+        <state relative-caret-position="750">
+          <caret line="50" column="0" lean-forward="false" selection-start-line="48" selection-start-column="8" selection-end-line="50" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/train.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/cf_gan.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="136">
-          <caret line="168" column="0" lean-forward="false" selection-start-line="168" selection-start-column="0" selection-end-line="168" selection-end-column="0" />
-          <folding>
-            <element signature="e#46#64#0" expanded="false" />
-          </folding>
+        <state relative-caret-position="2670">
+          <caret line="178" column="0" lean-forward="true" selection-start-line="178" selection-start-column="0" selection-end-line="178" selection-end-column="0" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/IO/ffm_cache.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DeepFM-model/main.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="31">
-          <caret line="67" column="29" lean-forward="false" selection-start-line="67" selection-start-column="29" selection-end-line="67" selection-end-column="29" />
-          <folding />
+        <state relative-caret-position="1665">
+          <caret line="122" column="0" lean-forward="false" selection-start-line="122" selection-start-column="0" selection-end-line="141" selection-end-column="1" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/IO/base_cache.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DeepFM-model/DataReader.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="195">
-          <caret line="13" column="64" lean-forward="false" selection-start-line="13" selection-start-column="64" selection-end-line="13" selection-end-column="64" />
+        <state relative-caret-position="0">
+          <caret line="30" column="32" lean-forward="false" selection-start-line="30" selection-start-column="32" selection-end-line="30" selection-end-column="32" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/IO/iterator.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DeepFM-model/DeepFM.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="2224">
-          <caret line="207" column="0" lean-forward="false" selection-start-line="207" selection-start-column="0" selection-end-line="207" selection-end-column="0" />
+        <state relative-caret-position="15">
+          <caret line="143" column="30" lean-forward="false" selection-start-line="136" selection-start-column="12" selection-end-line="143" selection-end-column="30" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/src/base_model.py">
+    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/__init__.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="2745">
-          <caret line="183" column="78" lean-forward="false" selection-start-line="183" selection-start-column="78" selection-end-line="183" selection-end-column="78" />
+        <state relative-caret-position="285">
+          <caret line="27" column="0" lean-forward="false" selection-start-line="27" selection-start-column="0" selection-end-line="27" selection-end-column="0" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/cache/eval_num.csv" />
-    <entry file="file://$PROJECT_DIR$/recommendation/Newton Interpolation.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/recommendation-FM-demo/FM_model.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="0">
-          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
-          <folding>
-            <element signature="e#0#18#0" expanded="false" />
-          </folding>
+        <state relative-caret-position="-819">
+          <caret line="15" column="26" lean-forward="false" selection-start-line="15" selection-start-column="26" selection-end-line="15" selection-end-column="26" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/multi-value-feature.py">
+    <entry file="file://$PROJECT_DIR$/CV/CNN.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="0">
-          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+        <state relative-caret-position="1230">
+          <caret line="83" column="23" lean-forward="false" selection-start-line="83" selection-start-column="23" selection-end-line="83" selection-end-column="23" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/utils/util.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo.py" />
+    <entry file="file://$PROJECT_DIR$/CV/ResNet/resnet_mnist.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="270">
-          <caret line="21" column="0" lean-forward="false" selection-start-line="21" selection-start-column="0" selection-end-line="21" selection-end-column="14" />
+        <state relative-caret-position="0">
+          <caret line="3" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="81" selection-end-column="0" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/config/network.yaml">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/data/cat_voc.pkl">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="90">
-          <caret line="6" column="28" lean-forward="false" selection-start-line="6" selection-start-column="28" selection-end-line="6" selection-end-column="28" />
+        <state relative-caret-position="375">
+          <caret line="25" column="18" lean-forward="false" selection-start-line="25" selection-start-column="18" selection-end-line="25" selection-end-column="18" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/.gitignore">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/shuffle.py" />
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIN-Demo/dataset.pkl">
+      <provider selected="true" editor-type-id="LargeFileEditor">
+        <state relative-caret-position="-1321">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/embedding_ops.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="285">
-          <caret line="19" column="11" lean-forward="false" selection-start-line="19" selection-start-column="11" selection-end-line="19" selection-end-column="11" />
+        <state relative-caret-position="184">
+          <caret line="262" column="4" lean-forward="false" selection-start-line="262" selection-start-column="4" selection-end-line="262" selection-end-column="4" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/test_some_code/tf_practice.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIN-Demo/Dice.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="0">
-          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="18" selection-end-column="0" />
-          <folding>
-            <element signature="e#0#23#0" expanded="true" />
-          </folding>
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/main.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIN-Demo/model.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="655">
-          <caret line="143" column="29" lean-forward="false" selection-start-line="143" selection-start-column="29" selection-end-line="143" selection-end-column="29" />
+        <state relative-caret-position="-261">
+          <caret line="29" column="45" lean-forward="false" selection-start-line="29" selection-start-column="45" selection-end-line="29" selection-end-column="45" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/xdeepfm-data/dnn/infer.userid.txt">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIN-Demo/train.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-615">
-          <caret line="0" column="2" lean-forward="true" selection-start-line="0" selection-start-column="2" selection-end-line="0" selection-end-column="10" />
-          <folding />
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding>
+            <element signature="e#0#13#0" expanded="false" />
+          </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/data/reviews-info">
+      <provider selected="true" editor-type-id="LargeFileEditor">
+        <state relative-caret-position="-471401">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/nltk/decorators.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-295">
-          <caret line="2370" column="4" lean-forward="false" selection-start-line="2370" selection-start-column="4" selection-end-line="2370" selection-end-column="4" />
+        <state relative-caret-position="223">
+          <caret line="131" column="8" lean-forward="false" selection-start-line="131" selection-start-column="8" selection-end-line="131" selection-end-column="8" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/basic/maxout.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/utils.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="165">
-          <caret line="11" column="15" lean-forward="false" selection-start-line="11" selection-start-column="15" selection-end-line="11" selection-end-column="15" />
+        <state relative-caret-position="561">
+          <caret line="58" column="0" lean-forward="false" selection-start-line="58" selection-start-column="0" selection-end-line="58" selection-end-column="0" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-XDeepFM-Demo/src/exDeepFM.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/vecAttGruCell.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-630">
-          <caret line="209" column="40" lean-forward="true" selection-start-line="209" selection-start-column="40" selection-end-line="209" selection-end-column="40" />
+        <state relative-caret-position="90">
+          <caret line="6" column="43" lean-forward="false" selection-start-line="6" selection-start-column="43" selection-end-line="6" selection-end-column="43" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/ml-100k/model_dns_ori.pkl">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/Dice.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="0">
-          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+        <state relative-caret-position="345">
+          <caret line="23" column="0" lean-forward="false" selection-start-line="23" selection-start-column="0" selection-end-line="23" selection-end-column="0" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/utils.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/rnn.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="1575">
-          <caret line="105" column="0" lean-forward="true" selection-start-line="105" selection-start-column="0" selection-end-line="105" selection-end-column="0" />
+        <state relative-caret-position="161">
+          <caret line="552" column="0" lean-forward="false" selection-start-line="552" selection-start-column="0" selection-end-line="552" selection-end-column="0" />
           <folding>
-            <element signature="e#0#16#0" expanded="true" />
+            <element signature="e#852#890#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/gen_model.py">
+    <entry file="file://$USER_HOME$/Anaconda/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="452">
-          <caret line="54" column="0" lean-forward="false" selection-start-line="54" selection-start-column="0" selection-end-line="54" selection-end-column="0" />
-          <folding>
-            <element signature="e#0#23#0" expanded="true" />
-          </folding>
+        <state relative-caret-position="5">
+          <caret line="194" column="71" lean-forward="false" selection-start-line="194" selection-start-column="71" selection-end-line="194" selection-end-column="71" />
+          <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/dis_model.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="437">
-          <caret line="60" column="0" lean-forward="false" selection-start-line="60" selection-start-column="0" selection-end-line="60" selection-end-column="0" />
-          <folding>
-            <element signature="e#0#23#0" expanded="true" />
-          </folding>
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/data/local_train_splitByUser">
+      <provider selected="true" editor-type-id="LargeFileEditor">
+        <state relative-caret-position="0">
+          <caret line="0" column="8" lean-forward="false" selection-start-line="0" selection-start-column="8" selection-end-line="0" selection-end-column="8" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/recommendation/Basic-IRGAN-Demo/cf_gan.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/data_iterator.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="350">
-          <caret line="199" column="5" lean-forward="true" selection-start-line="199" selection-start-column="5" selection-end-line="199" selection-end-column="5" />
+        <state relative-caret-position="157">
+          <caret line="137" column="23" lean-forward="false" selection-start-line="137" selection-start-column="23" selection-end-line="137" selection-end-column="23" />
           <folding>
-            <element signature="e#0#23#0" expanded="true" />
+            <element signature="e#0#18#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/GAN/GAN.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/model.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="180">
-          <caret line="23" column="4" lean-forward="false" selection-start-line="23" selection-start-column="4" selection-end-line="23" selection-end-column="4" />
+        <state relative-caret-position="346">
+          <caret line="88" column="17" lean-forward="false" selection-start-line="88" selection-start-column="17" selection-end-line="88" selection-end-column="17" />
           <folding>
-            <element signature="e#0#23#0" expanded="true" />
+            <element signature="e#0#18#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/RL/Basic-Policy-Network/RL_brain.py">
+    <entry file="file://$PROJECT_DIR$/recommendation/Basic-DIEN-Demo/train.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-744">
-          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
-          <folding />
+        <state relative-caret-position="-88">
+          <caret line="115" column="142" lean-forward="false" selection-start-line="115" selection-start-column="142" selection-end-line="115" selection-end-column="142" />
+          <folding>
+            <element signature="e#0#18#0" expanded="true" />
+          </folding>
         </state>
       </provider>
     </entry>
diff --git a/CV/ResNet/resnet_mnist.py b/CV/ResNet/resnet_mnist.py
index 1cc55d1d..9ea2ff79 100644
--- a/CV/ResNet/resnet_mnist.py
+++ b/CV/ResNet/resnet_mnist.py
@@ -58,7 +58,7 @@ def train():
     saver = tf.train.Saver()
     with tf.Session() as sess:
         sess.run(tf.global_variables_initializer())
-        for i in range(500):
+        for i in range(10000):
             x_b, y_b = mnist.train.next_batch(batch_size)
             train_op_, loss_, step = sess.run([train_op, loss, global_step], feed_dict={x: x_b, y: y_b})
             if i % 50 == 0:
diff --git a/recommendation/Basic-DIEN-Demo/Dice.py b/recommendation/Basic-DIEN-Demo/Dice.py
new file mode 100644
index 00000000..fa5c78ee
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/Dice.py
@@ -0,0 +1,25 @@
+import tensorflow as tf
+
+def dice(_x,axis=-1,epsilon=0.000000001,name=""):
+    with tf.variable_scope(name,reuse=tf.AUTO_REUSE):
+        alphas = tf.get_variable('alpha'+name,_x.get_shape()[-1],initializer=tf.constant_initializer(0.0),dtype=tf.float32)
+        input_shape = list(_x.get_shape())
+
+        reduction_axis = list(range(len(input_shape)))
+        del reduction_axis[axis]
+        broadcast_shape = [1] * len(input_shape)
+        broadcast_shape[axis] = input_shape[axis]
+
+    mean = tf.reduce_mean(_x, axis=reduction_axis)
+    brodcast_mean = tf.reshape(mean, broadcast_shape)
+    std = tf.reduce_mean(tf.square(_x - brodcast_mean) + epsilon, axis=reduction_axis)
+    std = tf.sqrt(std)
+    brodcast_std = tf.reshape(std, broadcast_shape)
+    x_normed = (_x - brodcast_mean) / (brodcast_std + epsilon)
+    # x_normed = tf.layers.batch_normalization(_x, center=False, scale=False)
+    x_p = tf.sigmoid(x_normed)
+
+    return alphas * (1.0 - x_p) * _x + x_p * _x
+
+
+
diff --git a/recommendation/Basic-DIEN-Demo/data_iterator.py b/recommendation/Basic-DIEN-Demo/data_iterator.py
new file mode 100644
index 00000000..6411105a
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/data_iterator.py
@@ -0,0 +1,208 @@
+import numpy as np
+import json
+import pickle as pkl
+import random
+import gzip
+
+
+def load_dict(filename):
+    try:
+        with open(filename,'r') as f:
+            return json.load(f)
+    except:
+        with open(filename,'rb') as f:
+            return pkl.load(f)
+
+def fopen(filename,mode='r'):
+    if filename.endswith('.gz'):
+        return gzip.open(filename,mode)
+    return open(filename,mode)
+
+
+class DataIterator:
+
+    def __init__(self,source,
+                 uid_voc,
+                 mid_voc,
+                 cat_voc,
+                 batch_size = 128,
+                 maxlen = 100,
+                 skip_empty = False,
+                 sort_by_length = True,
+                 max_batch_size = 20,
+                 minlen = None):
+        self.source = fopen(source,'r')
+
+        self.source_dicts = []
+        for source_dict in [uid_voc,mid_voc,cat_voc]:
+            self.source_dicts.append(load_dict(source_dict))
+
+        item_info = open("data/item-info",'r')
+        meta_map = {}
+        for line in item_info:
+            arr = line.strip().split("\t")
+            if arr[0] not in meta_map:
+                meta_map[arr[0]] = arr[1] # meta对应的类别map
+
+        self.meta_id_map = {}
+        for key in meta_map:
+            val = meta_map[key]
+            if key in self.source_dicts[1]:
+                mid_idx = self.source_dicts[1][key]
+            else:
+                mid_idx = 0
+            if val in self.source_dicts[2]:
+                cat_idx = self.source_dicts[2][val]
+            else:
+                cat_idx = 0
+            self.meta_id_map[mid_idx] = cat_idx # 得到meta id对应的cat id
+
+
+        review_info = open("data/reviews-info",'r')
+        self.mid_list_for_random = []
+        for line in review_info:
+            arr = line.strip().split("\t")
+            tmp_idx = 0
+            if arr[1] in self.source_dicts[1]:
+                tmp_idx = self.source_dicts[1][arr[1]]
+            self.mid_list_for_random.append(tmp_idx)
+
+        self.batch_size = batch_size
+        self.maxlen = maxlen
+        self.minlen = minlen
+        self.skip_empty = skip_empty
+
+        self.n_uid = len(self.source_dicts[0])
+        self.n_mid = len(self.source_dicts[1])
+        self.n_cat = len(self.source_dicts[2])
+
+        self.sort_by_length = sort_by_length
+        self.source_buffer = []
+        self.k = batch_size * max_batch_size
+        self.end_of_data = False
+
+    def get_n(self):
+        return self.n_uid, self.n_mid, self.n_cat
+
+    def reset(self):
+        self.source.seek(0)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self.end_of_data:
+            self.end_of_data = False
+            self.reset()
+            raise StopIteration
+
+        source = []
+        target = []
+
+        if len(self.source_buffer) == 0:
+            for k_ in range(self.k):
+                ss = self.source.readline()
+                if ss == "":
+                    break
+                self.source_buffer.append(ss.strip("\n").split('\t'))
+
+            if self.sort_by_length:
+                his_length = np.array([len(s[4].split(" ")) for s in self.source_buffer])
+                tidx = his_length.argsort()
+                _sbuf = [self.source_buffer[i] for i in tidx]
+                self.source_buffer = _sbuf
+            else:
+                self.source_buffer.reverse()
+
+
+        if len(self.source_buffer) == 0:
+            self.end_of_data = False
+            self.reset()
+            raise StopIteration
+        try:
+            while True:
+
+                # 将数据都转换为id
+                try:
+                    ss = self.source_buffer.pop()
+                except IndexError:
+                    break
+
+                uid = self.source_dicts[0][ss[1]] if ss[1] in self.source_dicts[0] else 0
+                mid = self.source_dicts[1][ss[2]] if ss[2] in self.source_dicts[1] else 0
+                cat = self.source_dicts[2][ss[3]] if ss[3] in self.source_dicts[2] else 0
+
+                tmp = []
+                for fea in ss[4].split(""):
+                    m = self.source_dicts[1][fea] if fea in self.source_dicts[1] else 0
+                    tmp.append(m)
+                mid_list = tmp
+
+                tmp1 = []
+                for fea in ss[5].split(""):
+                    c = self.source_dicts[2][fea] if fea in self.source_dicts[2] else 0
+                    tmp1.append(c)
+                cat_list = tmp1
+
+                # 如果小于规定的最小历史记录长度，直接过滤
+                if self.minlen != None:
+                    if len(mid_list) >= self.minlen:
+                        continue
+                if self.skip_empty and (not mid_list):
+                    continue
+
+                noclk_mid_list = []
+                noclk_cat_list = []
+
+                for pos_mid in mid_list:
+                    noclk_tmp_mid = []
+                    noclk_tmp_cat = []
+                    noclk_index = 0
+
+                    while True:
+                        noclk_mid_indx = random.randint(0, len(self.mid_list_for_random) - 1)
+                        noclk_mid = self.mid_list_for_random[noclk_mid_indx]
+                        if noclk_mid == pos_mid:
+                            continue
+                        noclk_tmp_mid.append(noclk_mid)
+                        noclk_tmp_cat.append(self.meta_id_map[noclk_mid])
+                        noclk_index += 1
+                        if noclk_index >= 5:
+                            break
+
+                    noclk_mid_list.append(noclk_tmp_mid)
+                    noclk_cat_list.append(noclk_tmp_cat)
+
+                source.append([uid,mid,cat,mid_list,cat_list,noclk_mid_list,noclk_cat_list])
+                target.append([float(ss[0]), 1 - float(ss[0])])
+
+                if len(source) >= self.batch_size or len(target) >= self.batch_size:
+                    break
+        except IOError:
+            self.end_of_data = True
+
+        # all sentence pairs in maxibatch filtered out because of length
+        if len(source) == 0 or len(target) == 0:
+            source, target = self.__next__()
+
+
+        return source,target
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/recommendation/Basic-DIEN-Demo/model.py b/recommendation/Basic-DIEN-Demo/model.py
new file mode 100644
index 00000000..b929c400
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/model.py
@@ -0,0 +1,224 @@
+import numpy as np
+import tensorflow as tf
+from tensorflow.python.ops.rnn_cell import GRUCell
+
+from rnn import dynamic_rnn
+
+from Dice import dice
+from utils import prelu,din_fcn_attention
+from vecAttGruCell import VecAttGRUCell
+
+
+
+
+class Model(object):
+    def __init__(self,n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling = False):
+        with tf.name_scope("Inputs"):
+            self.mid_his_batch_ph = tf.placeholder(tf.int32,[None,None],name='mid_his_batch_ph')
+            self.cat_his_batch_ph = tf.placeholder(tf.int32,[None,None],name='cat_his_batch_ph')
+            self.uid_batch_ph = tf.placeholder(tf.int32,[None,],name='uid_batch_ph')
+            self.mid_batch_ph = tf.placeholder(tf.int32,[None,],name='mid_batch_ph')
+            self.cat_batch_ph = tf.placeholder(tf.int32,[None,],name='cat_batch_ph')
+            self.mask = tf.placeholder(tf.float32,[None,None],name='mask')
+            self.seq_len_ph = tf.placeholder(tf.int32,[None],name='seq_len_ph')
+            self.target_ph = tf.placeholder(tf.float32,[None,None],name='target_ph')
+            self.lr = tf.placeholder(tf.float64,[])
+            self.use_negsampling = use_negsampling
+            if use_negsampling:
+                self.noclk_mid_batch_ph = tf.placeholder(tf.int32, [None, None, None], name='noclk_mid_batch_ph')
+                self.noclk_cat_batch_ph = tf.placeholder(tf.int32, [None, None, None], name='noclk_cat_batch_ph')
+
+        with tf.name_scope("Embedding_layer"):
+            self.uid_embeddings_var = tf.get_variable("uid_embedding_var",[n_uid,EMBEDDING_DIM])
+            tf.summary.histogram('uid_embeddings_var', self.uid_embeddings_var)
+            self.uid_batch_embedded = tf.nn.embedding_lookup(self.uid_embeddings_var,self.uid_batch_ph)
+
+            self.mid_embeddings_var = tf.get_variable("mid_embedding_var",[n_mid,EMBEDDING_DIM])
+            tf.summary.histogram('mid_embeddings_var',self.mid_embeddings_var)
+            self.mid_batch_embedded = tf.nn.embedding_lookup(self.mid_embeddings_var,self.mid_batch_ph)
+            self.mid_his_batch_embedded = tf.nn.embedding_lookup(self.mid_embeddings_var,self.mid_his_batch_ph)
+            if self.use_negsampling:
+                self.noclk_mid_his_batch_embedded = tf.nn.embedding_lookup(self.mid_embeddings_var,
+                                                                           self.noclk_mid_batch_ph)
+
+            self.cat_embeddings_var = tf.get_variable("cat_embedding_var", [n_cat, EMBEDDING_DIM])
+            tf.summary.histogram('cat_embeddings_var', self.cat_embeddings_var)
+            self.cat_batch_embedded = tf.nn.embedding_lookup(self.cat_embeddings_var, self.cat_batch_ph)
+            self.cat_his_batch_embedded = tf.nn.embedding_lookup(self.cat_embeddings_var, self.cat_his_batch_ph)
+            if self.use_negsampling:
+                self.noclk_cat_his_batch_embedded = tf.nn.embedding_lookup(self.cat_embeddings_var,
+                                                                           self.noclk_cat_batch_ph)
+
+
+        self.item_eb = tf.concat([self.mid_batch_embedded,self.cat_batch_embedded],1)
+        self.item_his_eb = tf.concat([self.mid_his_batch_embedded,self.cat_his_batch_embedded],2)
+        self.item_his_eb_sum = tf.reduce_sum(self.item_his_eb,1)
+
+        if self.use_negsampling:
+            self.noclk_item_his_eb = tf.concat(
+                [self.noclk_mid_his_batch_embedded[:, :, 0, :], self.noclk_cat_his_batch_embedded[:, :, 0, :]], -1)
+            self.noclk_item_his_eb = tf.reshape(self.noclk_item_his_eb,
+                                                [-1, tf.shape(self.noclk_mid_his_batch_embedded)[1], EMBEDDING_DIM * 2]) # 负采样的item选第一个
+
+            self.noclk_his_eb = tf.concat([self.noclk_mid_his_batch_embedded, self.noclk_cat_his_batch_embedded], -1)
+            self.noclk_his_eb_sum_1 = tf.reduce_sum(self.noclk_his_eb, 2)
+            self.noclk_his_eb_sum = tf.reduce_sum(self.noclk_his_eb_sum_1, 1) # 负采样的样本
+
+
+
+    def build_fcn_net(self,inp,use_dice=False):
+        bn1 = tf.layers.batch_normalization(inputs=inp,name='bn1')
+        dnn1 = tf.layers.dense(bn1,200,activation=None,name='f1')
+
+        if use_dice:
+            dnn1 = dice(dnn1,name='dice_1')
+        else:
+            dnn1 = prelu(dnn1,'prelu1')
+
+        dnn2 = tf.layers.dense(dnn1,80,activation=None,name='f2')
+        if use_dice:
+            dnn2 = dice(dnn2,name='dice_2')
+        else:
+            dnn2 = prelu(dnn2,name='prelu2')
+
+        dnn3 = tf.layers.dense(dnn2,2,activation=None,name='f3')
+        self.y_hat = tf.nn.softmax(dnn3) + 0.00000001
+
+        with tf.name_scope('Metrics'):
+            ctr_loss = -tf.reduce_mean(tf.log(self.y_hat) * self.target_ph)
+            self.loss = ctr_loss
+            if self.use_negsampling:
+                self.loss += self.aux_loss
+            tf.summary.scalar('loss',self.loss)
+            self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(self.loss)
+
+            self.accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.round(self.y_hat),self.target_ph),tf.float32))
+            tf.summary.scalar('accuracy',self.accuracy)
+
+        self.merged = tf.summary.merge_all()
+
+
+    def auxiliary_loss(self,h_states,click_seq,noclick_seq,mask,stag=None):
+        mask = tf.cast(mask,tf.float32)
+        click_input = tf.concat([h_states,click_seq],-1)
+        noclick_input = tf.concat([h_states,noclick_seq],-1)
+        click_prop_ = self.auxiliary_net(click_input,stag=stag)[:,:,0]
+        noclick_prop_ = self.auxiliary_net(noclick_input,stag=stag)[:,:,0]
+        click_loss_ = -tf.reshape(tf.log(click_prop_),[-1,tf.shape(click_seq)[1]]) * mask
+        noclick_loss_ = - tf.reshape(tf.log(1.0 - noclick_prop_), [-1, tf.shape(noclick_seq)[1]]) * mask
+        loss_ = tf.reduce_mean(click_loss_ + noclick_loss_)
+        return loss_
+
+
+
+    def auxiliary_net(self,input,stag='auxiliary_net'):
+        bn1 = tf.layers.batch_normalization(inputs=input, name='bn1' + stag, reuse=tf.AUTO_REUSE)
+        dnn1 = tf.layers.dense(bn1, 100, activation=None, name='f1' + stag, reuse=tf.AUTO_REUSE)
+        dnn1 = tf.nn.sigmoid(dnn1)
+        dnn2 = tf.layers.dense(dnn1, 50, activation=None, name='f2' + stag, reuse=tf.AUTO_REUSE)
+        dnn2 = tf.nn.sigmoid(dnn2)
+        dnn3 = tf.layers.dense(dnn2, 2, activation=None, name='f3' + stag, reuse=tf.AUTO_REUSE)
+        y_hat = tf.nn.softmax(dnn3) + 0.00000001
+        return y_hat
+
+    def train(self, sess, inps):
+        if self.use_negsampling:
+            loss, accuracy, aux_loss, _ = sess.run([self.loss, self.accuracy, self.aux_loss, self.optimizer],
+                                                   feed_dict={
+                                                       self.uid_batch_ph: inps[0],
+                                                       self.mid_batch_ph: inps[1],
+                                                       self.cat_batch_ph: inps[2],
+                                                       self.mid_his_batch_ph: inps[3],
+                                                       self.cat_his_batch_ph: inps[4],
+                                                       self.mask: inps[5],
+                                                       self.target_ph: inps[6],
+                                                       self.seq_len_ph: inps[7],
+                                                       self.lr: inps[8],
+                                                       self.noclk_mid_batch_ph: inps[9],
+                                                       self.noclk_cat_batch_ph: inps[10],
+                                                   })
+            return loss, accuracy, aux_loss
+        else:
+            loss, accuracy, _ = sess.run([self.loss, self.accuracy, self.optimizer], feed_dict={
+                self.uid_batch_ph: inps[0],
+                self.mid_batch_ph: inps[1],
+                self.cat_batch_ph: inps[2],
+                self.mid_his_batch_ph: inps[3],
+                self.cat_his_batch_ph: inps[4],
+                self.mask: inps[5],
+                self.target_ph: inps[6],
+                self.seq_len_ph: inps[7],
+                self.lr: inps[8],
+            })
+            return loss, accuracy, 0
+
+    def calculate(self, sess, inps):
+        if self.use_negsampling:
+            probs, loss, accuracy, aux_loss = sess.run([self.y_hat, self.loss, self.accuracy, self.aux_loss],
+                                                       feed_dict={
+                                                           self.uid_batch_ph: inps[0],
+                                                           self.mid_batch_ph: inps[1],
+                                                           self.cat_batch_ph: inps[2],
+                                                           self.mid_his_batch_ph: inps[3],
+                                                           self.cat_his_batch_ph: inps[4],
+                                                           self.mask: inps[5],
+                                                           self.target_ph: inps[6],
+                                                           self.seq_len_ph: inps[7],
+                                                           self.noclk_mid_batch_ph: inps[8],
+                                                           self.noclk_cat_batch_ph: inps[9],
+                                                       })
+            return probs, loss, accuracy, aux_loss
+        else:
+            probs, loss, accuracy = sess.run([self.y_hat, self.loss, self.accuracy], feed_dict={
+                self.uid_batch_ph: inps[0],
+                self.mid_batch_ph: inps[1],
+                self.cat_batch_ph: inps[2],
+                self.mid_his_batch_ph: inps[3],
+                self.cat_his_batch_ph: inps[4],
+                self.mask: inps[5],
+                self.target_ph: inps[6],
+                self.seq_len_ph: inps[7]
+            })
+            return probs, loss, accuracy, 0
+
+    def save(self, sess, path):
+        saver = tf.train.Saver()
+        saver.save(sess, save_path=path)
+
+    def restore(self, sess, path):
+        saver = tf.train.Saver()
+        saver.restore(sess, save_path=path)
+        print('model restored from %s' % path)
+
+
+
+class Model_DIN_V2_Gru_Vec_attGru_Neg(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=True):
+        super(Model_DIN_V2_Gru_Vec_attGru_Neg, self).__init__(n_uid, n_mid, n_cat,EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE,use_negsampling)
+
+        with tf.name_scope('rnn_1'):
+            rnn_outputs,_ = dynamic_rnn(GRUCell(HIDDEN_SIZE),inputs = self.item_his_eb,sequence_length=self.seq_len_ph,dtype=tf.float32,scope='gru1')
+            tf.summary.histogram("GRU_outputs",rnn_outputs)
+
+        aux_loss_1 = self.auxiliary_loss(rnn_outputs[:,:-1,:],self.item_his_eb[:,1:,:],self.noclk_item_his_eb[:,1:,:],self.mask[:,1:],stag="gru")
+        self.aux_loss = aux_loss_1
+        with tf.name_scope('Attention_layer_1'):
+            att_outputs,alphas = din_fcn_attention(self.item_eb,rnn_outputs,ATTENTION_SIZE,self.mask,
+                                                   softmax_stag=1,stag='1_1',mode='LIST',return_alphas=True)
+
+            tf.summary.histogram('alpha_outputs',alphas)
+
+        with tf.name_scope('rnn_2'):
+            rnn_outputs2,final_state2 = dynamic_rnn(VecAttGRUCell(HIDDEN_SIZE),inputs=rnn_outputs,
+                                                    att_scores=tf.expand_dims(alphas,-1),
+                                                    sequence_length = self.seq_len_ph,dtype=tf.float32,
+                                                    scope="gru2"
+                                                    )
+            tf.summary.histogram("GRU2_Final_State",final_state2)
+
+        inp = tf.concat([self.uid_batch_embedded,self.item_eb,self.item_his_eb_sum,self.item_eb * self.item_his_eb_sum,final_state2],1)
+        self.build_fcn_net(inp,use_dice=True)
+
+
+
+
diff --git a/recommendation/Basic-DIEN-Demo/rnn.py b/recommendation/Basic-DIEN-Demo/rnn.py
new file mode 100644
index 00000000..0a82a92b
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/rnn.py
@@ -0,0 +1,1456 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""RNN helpers for TensorFlow models.
+
+
+@@bidirectional_dynamic_rnn
+@@dynamic_rnn
+@@raw_rnn
+@@static_rnn
+@@static_state_saving_rnn
+@@static_bidirectional_rnn
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import rnn_cell_impl
+from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.util import nest
+
+# pylint: disable=protected-access
+_concat = rnn_cell_impl._concat
+_like_rnncell = rnn_cell_impl._like_rnncell
+
+
+# pylint: enable=protected-access
+
+
+def _transpose_batch_time(x):
+    """Transpose the batch and time dimensions of a Tensor.
+
+    Retains as much of the static shape information as possible.
+
+    Args:
+      x: A tensor of rank 2 or higher.
+
+    Returns:
+      x transposed along the first two dimensions.
+
+    Raises:
+      ValueError: if `x` is rank 1 or lower.
+    """
+    x_static_shape = x.get_shape()
+    if x_static_shape.ndims is not None and x_static_shape.ndims < 2:
+        raise ValueError(
+            "Expected input tensor %s to have rank at least 2, but saw shape: %s" %
+            (x, x_static_shape))
+    x_rank = array_ops.rank(x)
+    x_t = array_ops.transpose(
+        x, array_ops.concat(
+            ([1, 0], math_ops.range(2, x_rank)), axis=0))
+    x_t.set_shape(
+        tensor_shape.TensorShape([
+            x_static_shape[1].value, x_static_shape[0].value
+        ]).concatenate(x_static_shape[2:]))
+    return x_t
+
+
+def _best_effort_input_batch_size(flat_input):
+    """Get static input batch size if available, with fallback to the dynamic one.
+
+    Args:
+      flat_input: An iterable of time major input Tensors of shape [max_time,
+        batch_size, ...]. All inputs should have compatible batch sizes.
+
+    Returns:
+      The batch size in Python integer if available, or a scalar Tensor otherwise.
+
+    Raises:
+      ValueError: if there is any input with an invalid shape.
+    """
+    for input_ in flat_input:
+        shape = input_.shape
+        if shape.ndims is None:
+            continue
+        if shape.ndims < 2:
+            raise ValueError(
+                "Expected input tensor %s to have rank at least 2" % input_)
+        batch_size = shape[1].value
+        if batch_size is not None:
+            return batch_size
+    # Fallback to the dynamic batch size of the first input.
+    return array_ops.shape(flat_input[0])[1]
+
+
+def _infer_state_dtype(explicit_dtype, state):
+    """Infer the dtype of an RNN state.
+
+    Args:
+      explicit_dtype: explicitly declared dtype or None.
+      state: RNN's hidden state. Must be a Tensor or a nested iterable containing
+        Tensors.
+
+    Returns:
+      dtype: inferred dtype of hidden state.
+
+    Raises:
+      ValueError: if `state` has heterogeneous dtypes or is empty.
+    """
+    if explicit_dtype is not None:
+        return explicit_dtype
+    elif nest.is_sequence(state):
+        inferred_dtypes = [element.dtype for element in nest.flatten(state)]
+        if not inferred_dtypes:
+            raise ValueError("Unable to infer dtype from empty state.")
+        all_same = all([x == inferred_dtypes[0] for x in inferred_dtypes])
+        if not all_same:
+            raise ValueError(
+                "State has tensors of different inferred_dtypes. Unable to infer a "
+                "single representative dtype.")
+        return inferred_dtypes[0]
+    else:
+        return state.dtype
+
+
+# pylint: disable=unused-argument
+def _rnn_step(
+        time, sequence_length, min_sequence_length, max_sequence_length,
+        zero_output, state, call_cell, state_size, skip_conditionals=False):
+    """Calculate one step of a dynamic RNN minibatch.
+
+    Returns an (output, state) pair conditioned on the sequence_lengths.
+    When skip_conditionals=False, the pseudocode is something like:
+
+    if t >= max_sequence_length:
+      return (zero_output, state)
+    if t < min_sequence_length:
+      return call_cell()
+
+    # Selectively output zeros or output, old state or new state depending
+    # on if we've finished calculating each row.
+    new_output, new_state = call_cell()
+    final_output = np.vstack([
+      zero_output if time >= sequence_lengths[r] else new_output_r
+      for r, new_output_r in enumerate(new_output)
+    ])
+    final_state = np.vstack([
+      state[r] if time >= sequence_lengths[r] else new_state_r
+      for r, new_state_r in enumerate(new_state)
+    ])
+    return (final_output, final_state)
+
+    Args:
+      time: Python int, the current time step
+      sequence_length: int32 `Tensor` vector of size [batch_size]
+      min_sequence_length: int32 `Tensor` scalar, min of sequence_length
+      max_sequence_length: int32 `Tensor` scalar, max of sequence_length
+      zero_output: `Tensor` vector of shape [output_size]
+      state: Either a single `Tensor` matrix of shape `[batch_size, state_size]`,
+        or a list/tuple of such tensors.
+      call_cell: lambda returning tuple of (new_output, new_state) where
+        new_output is a `Tensor` matrix of shape `[batch_size, output_size]`.
+        new_state is a `Tensor` matrix of shape `[batch_size, state_size]`.
+      state_size: The `cell.state_size` associated with the state.
+      skip_conditionals: Python bool, whether to skip using the conditional
+        calculations.  This is useful for `dynamic_rnn`, where the input tensor
+        matches `max_sequence_length`, and using conditionals just slows
+        everything down.
+
+    Returns:
+      A tuple of (`final_output`, `final_state`) as given by the pseudocode above:
+        final_output is a `Tensor` matrix of shape [batch_size, output_size]
+        final_state is either a single `Tensor` matrix, or a tuple of such
+          matrices (matching length and shapes of input `state`).
+
+    Raises:
+      ValueError: If the cell returns a state tuple whose length does not match
+        that returned by `state_size`.
+    """
+
+    # Convert state to a list for ease of use
+    flat_state = nest.flatten(state)
+    flat_zero_output = nest.flatten(zero_output)
+
+    def _copy_one_through(output, new_output):
+        # If the state contains a scalar value we simply pass it through.
+        if output.shape.ndims == 0:
+            return new_output
+        copy_cond = (time >= sequence_length)
+        with ops.colocate_with(new_output):
+            return array_ops.where(copy_cond, output, new_output)
+
+    def _copy_some_through(flat_new_output, flat_new_state):
+        # Use broadcasting select to determine which values should get
+        # the previous state & zero output, and which values should get
+        # a calculated state & output.
+        flat_new_output = [
+            _copy_one_through(zero_output, new_output)
+            for zero_output, new_output in zip(flat_zero_output, flat_new_output)]
+        flat_new_state = [
+            _copy_one_through(state, new_state)
+            for state, new_state in zip(flat_state, flat_new_state)]
+        return flat_new_output + flat_new_state
+
+    def _maybe_copy_some_through():
+        """Run RNN step.  Pass through either no or some past state."""
+        new_output, new_state = call_cell()
+
+        nest.assert_same_structure(state, new_state)
+
+        flat_new_state = nest.flatten(new_state)
+        flat_new_output = nest.flatten(new_output)
+        return control_flow_ops.cond(
+            # if t < min_seq_len: calculate and return everything
+            time < min_sequence_length, lambda: flat_new_output + flat_new_state,
+            # else copy some of it through
+            lambda: _copy_some_through(flat_new_output, flat_new_state))
+
+    # TODO(ebrevdo): skipping these conditionals may cause a slowdown,
+    # but benefits from removing cond() and its gradient.  We should
+    # profile with and without this switch here.
+    if skip_conditionals:
+        # Instead of using conditionals, perform the selective copy at all time
+        # steps.  This is faster when max_seq_len is equal to the number of unrolls
+        # (which is typical for dynamic_rnn).
+        new_output, new_state = call_cell()
+        nest.assert_same_structure(state, new_state)
+        new_state = nest.flatten(new_state)
+        new_output = nest.flatten(new_output)
+        final_output_and_state = _copy_some_through(new_output, new_state)
+    else:
+        empty_update = lambda: flat_zero_output + flat_state
+        final_output_and_state = control_flow_ops.cond(
+            # if t >= max_seq_len: copy all state through, output zeros
+            time >= max_sequence_length, empty_update,
+            # otherwise calculation is required: copy some or all of it through
+            _maybe_copy_some_through)
+
+    if len(final_output_and_state) != len(flat_zero_output) + len(flat_state):
+        raise ValueError("Internal error: state and output were not concatenated "
+                         "correctly.")
+    final_output = final_output_and_state[:len(flat_zero_output)]
+    final_state = final_output_and_state[len(flat_zero_output):]
+
+    for output, flat_output in zip(final_output, flat_zero_output):
+        output.set_shape(flat_output.get_shape())
+    for substate, flat_substate in zip(final_state, flat_state):
+        substate.set_shape(flat_substate.get_shape())
+
+    final_output = nest.pack_sequence_as(
+        structure=zero_output, flat_sequence=final_output)
+    final_state = nest.pack_sequence_as(
+        structure=state, flat_sequence=final_state)
+
+    return final_output, final_state
+
+
+def _reverse_seq(input_seq, lengths):
+    """Reverse a list of Tensors up to specified lengths.
+
+    Args:
+      input_seq: Sequence of seq_len tensors of dimension (batch_size, n_features)
+                 or nested tuples of tensors.
+      lengths:   A `Tensor` of dimension batch_size, containing lengths for each
+                 sequence in the batch. If "None" is specified, simply reverses
+                 the list.
+
+    Returns:
+      time-reversed sequence
+    """
+    if lengths is None:
+        return list(reversed(input_seq))
+
+    flat_input_seq = tuple(nest.flatten(input_) for input_ in input_seq)
+
+    flat_results = [[] for _ in range(len(input_seq))]
+    for sequence in zip(*flat_input_seq):
+        input_shape = tensor_shape.unknown_shape(
+            ndims=sequence[0].get_shape().ndims)
+        for input_ in sequence:
+            input_shape.merge_with(input_.get_shape())
+            input_.set_shape(input_shape)
+
+        # Join into (time, batch_size, depth)
+        s_joined = array_ops.stack(sequence)
+
+        # Reverse along dimension 0
+        s_reversed = array_ops.reverse_sequence(s_joined, lengths, 0, 1)
+        # Split again into list
+        result = array_ops.unstack(s_reversed)
+        for r, flat_result in zip(result, flat_results):
+            r.set_shape(input_shape)
+            flat_result.append(r)
+
+    results = [nest.pack_sequence_as(structure=input_, flat_sequence=flat_result)
+               for input_, flat_result in zip(input_seq, flat_results)]
+    return results
+
+
+def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None,
+                              initial_state_fw=None, initial_state_bw=None,
+                              dtype=None, parallel_iterations=None,
+                              swap_memory=False, time_major=False, scope=None):
+    """Creates a dynamic version of bidirectional recurrent neural network.
+
+    Takes input and builds independent forward and backward RNNs. The input_size
+    of forward and backward cell must match. The initial state for both directions
+    is zero by default (but can be set optionally) and no intermediate states are
+    ever returned -- the network is fully unrolled for the given (passed in)
+    length(s) of the sequence(s) or completely unrolled if length(s) is not
+    given.
+
+    Args:
+      cell_fw: An instance of RNNCell, to be used for forward direction.
+      cell_bw: An instance of RNNCell, to be used for backward direction.
+      inputs: The RNN inputs.
+        If time_major == False (default), this must be a tensor of shape:
+          `[batch_size, max_time, ...]`, or a nested tuple of such elements.
+        If time_major == True, this must be a tensor of shape:
+          `[max_time, batch_size, ...]`, or a nested tuple of such elements.
+      sequence_length: (optional) An int32/int64 vector, size `[batch_size]`,
+        containing the actual lengths for each of the sequences in the batch.
+        If not provided, all batch entries are assumed to be full sequences; and
+        time reversal is applied from time `0` to `max_time` for each sequence.
+      initial_state_fw: (optional) An initial state for the forward RNN.
+        This must be a tensor of appropriate type and shape
+        `[batch_size, cell_fw.state_size]`.
+        If `cell_fw.state_size` is a tuple, this should be a tuple of
+        tensors having shapes `[batch_size, s] for s in cell_fw.state_size`.
+      initial_state_bw: (optional) Same as for `initial_state_fw`, but using
+        the corresponding properties of `cell_bw`.
+      dtype: (optional) The data type for the initial states and expected output.
+        Required if initial_states are not provided or RNN states have a
+        heterogeneous dtype.
+      parallel_iterations: (Default: 32).  The number of iterations to run in
+        parallel.  Those operations which do not have any temporal dependency
+        and can be run in parallel, will be.  This parameter trades off
+        time for space.  Values >> 1 use more memory but take less time,
+        while smaller values use less memory but computations take longer.
+      swap_memory: Transparently swap the tensors produced in forward inference
+        but needed for back prop from GPU to CPU.  This allows training RNNs
+        which would typically not fit on a single GPU, with very minimal (or no)
+        performance penalty.
+      time_major: The shape format of the `inputs` and `outputs` Tensors.
+        If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
+        If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
+        Using `time_major = True` is a bit more efficient because it avoids
+        transposes at the beginning and end of the RNN calculation.  However,
+        most TensorFlow data is batch-major, so by default this function
+        accepts input and emits output in batch-major form.
+      scope: VariableScope for the created subgraph; defaults to
+        "bidirectional_rnn"
+
+    Returns:
+      A tuple (outputs, output_states) where:
+        outputs: A tuple (output_fw, output_bw) containing the forward and
+          the backward rnn output `Tensor`.
+          If time_major == False (default),
+            output_fw will be a `Tensor` shaped:
+            `[batch_size, max_time, cell_fw.output_size]`
+            and output_bw will be a `Tensor` shaped:
+            `[batch_size, max_time, cell_bw.output_size]`.
+          If time_major == True,
+            output_fw will be a `Tensor` shaped:
+            `[max_time, batch_size, cell_fw.output_size]`
+            and output_bw will be a `Tensor` shaped:
+            `[max_time, batch_size, cell_bw.output_size]`.
+          It returns a tuple instead of a single concatenated `Tensor`, unlike
+          in the `bidirectional_rnn`. If the concatenated one is preferred,
+          the forward and backward outputs can be concatenated as
+          `tf.concat(outputs, 2)`.
+        output_states: A tuple (output_state_fw, output_state_bw) containing
+          the forward and the backward final states of bidirectional rnn.
+
+    Raises:
+      TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`.
+    """
+
+    if not _like_rnncell(cell_fw):
+        raise TypeError("cell_fw must be an instance of RNNCell")
+    if not _like_rnncell(cell_bw):
+        raise TypeError("cell_bw must be an instance of RNNCell")
+
+    with vs.variable_scope(scope or "bidirectional_rnn"):
+        # Forward direction
+        with vs.variable_scope("fw") as fw_scope:
+            output_fw, output_state_fw = dynamic_rnn(
+                cell=cell_fw, inputs=inputs, sequence_length=sequence_length,
+                initial_state=initial_state_fw, dtype=dtype,
+                parallel_iterations=parallel_iterations, swap_memory=swap_memory,
+                time_major=time_major, scope=fw_scope)
+
+        # Backward direction
+        if not time_major:
+            time_dim = 1
+            batch_dim = 0
+        else:
+            time_dim = 0
+            batch_dim = 1
+
+        def _reverse(input_, seq_lengths, seq_dim, batch_dim):
+            if seq_lengths is not None:
+                return array_ops.reverse_sequence(
+                    input=input_, seq_lengths=seq_lengths,
+                    seq_dim=seq_dim, batch_dim=batch_dim)
+            else:
+                return array_ops.reverse(input_, axis=[seq_dim])
+
+        with vs.variable_scope("bw") as bw_scope:
+            inputs_reverse = _reverse(
+                inputs, seq_lengths=sequence_length,
+                seq_dim=time_dim, batch_dim=batch_dim)
+            tmp, output_state_bw = dynamic_rnn(
+                cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length,
+                initial_state=initial_state_bw, dtype=dtype,
+                parallel_iterations=parallel_iterations, swap_memory=swap_memory,
+                time_major=time_major, scope=bw_scope)
+
+    output_bw = _reverse(
+        tmp, seq_lengths=sequence_length,
+        seq_dim=time_dim, batch_dim=batch_dim)
+
+    outputs = (output_fw, output_bw)
+    output_states = (output_state_fw, output_state_bw)
+
+    return (outputs, output_states)
+
+
+def dynamic_rnn(cell, inputs, att_scores=None, sequence_length=None, initial_state=None,
+                dtype=None, parallel_iterations=None, swap_memory=False,
+                time_major=False, scope=None):
+    """Creates a recurrent neural network specified by RNNCell `cell`.
+
+    Performs fully dynamic unrolling of `inputs`.
+
+    Example:
+
+    ```python
+    # create a BasicRNNCell
+    rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
+
+    # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size]
+
+    # defining initial state
+    initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32)
+
+    # 'state' is a tensor of shape [batch_size, cell_state_size]
+    outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data,
+                                       initial_state=initial_state,
+                                       dtype=tf.float32)
+    ```
+
+    ```python
+    # create 2 LSTMCells
+    rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]]
+
+    # create a RNN cell composed sequentially of a number of RNNCells
+    multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)
+
+    # 'outputs' is a tensor of shape [batch_size, max_time, 256]
+    # 'state' is a N-tuple where N is the number of LSTMCells containing a
+    # tf.contrib.rnn.LSTMStateTuple for each cell
+    outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell,
+                                       inputs=data,
+                                       dtype=tf.float32)
+    ```
+
+
+    Args:
+      cell: An instance of RNNCell.
+      inputs: The RNN inputs.
+        If `time_major == False` (default), this must be a `Tensor` of shape:
+          `[batch_size, max_time, ...]`, or a nested tuple of such
+          elements.
+        If `time_major == True`, this must be a `Tensor` of shape:
+          `[max_time, batch_size, ...]`, or a nested tuple of such
+          elements.
+        This may also be a (possibly nested) tuple of Tensors satisfying
+        this property.  The first two dimensions must match across all the inputs,
+        but otherwise the ranks and other shape components may differ.
+        In this case, input to `cell` at each time-step will replicate the
+        structure of these tuples, except for the time dimension (from which the
+        time is taken).
+        The input to `cell` at each time step will be a `Tensor` or (possibly
+        nested) tuple of Tensors each with dimensions `[batch_size, ...]`.
+      sequence_length: (optional) An int32/int64 vector sized `[batch_size]`.
+        Used to copy-through state and zero-out outputs when past a batch
+        element's sequence length.  So it's more for correctness than performance.
+      initial_state: (optional) An initial state for the RNN.
+        If `cell.state_size` is an integer, this must be
+        a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`.
+        If `cell.state_size` is a tuple, this should be a tuple of
+        tensors having shapes `[batch_size, s] for s in cell.state_size`.
+      dtype: (optional) The data type for the initial state and expected output.
+        Required if initial_state is not provided or RNN state has a heterogeneous
+        dtype.
+      parallel_iterations: (Default: 32).  The number of iterations to run in
+        parallel.  Those operations which do not have any temporal dependency
+        and can be run in parallel, will be.  This parameter trades off
+        time for space.  Values >> 1 use more memory but take less time,
+        while smaller values use less memory but computations take longer.
+      swap_memory: Transparently swap the tensors produced in forward inference
+        but needed for back prop from GPU to CPU.  This allows training RNNs
+        which would typically not fit on a single GPU, with very minimal (or no)
+        performance penalty.
+      time_major: The shape format of the `inputs` and `outputs` Tensors.
+        If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
+        If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
+        Using `time_major = True` is a bit more efficient because it avoids
+        transposes at the beginning and end of the RNN calculation.  However,
+        most TensorFlow data is batch-major, so by default this function
+        accepts input and emits output in batch-major form.
+      scope: VariableScope for the created subgraph; defaults to "rnn".
+
+    Returns:
+      A pair (outputs, state) where:
+
+      outputs: The RNN output `Tensor`.
+
+        If time_major == False (default), this will be a `Tensor` shaped:
+          `[batch_size, max_time, cell.output_size]`.
+
+        If time_major == True, this will be a `Tensor` shaped:
+          `[max_time, batch_size, cell.output_size]`.
+
+        Note, if `cell.output_size` is a (possibly nested) tuple of integers
+        or `TensorShape` objects, then `outputs` will be a tuple having the
+        same structure as `cell.output_size`, containing Tensors having shapes
+        corresponding to the shape data in `cell.output_size`.
+
+      state: The final state.  If `cell.state_size` is an int, this
+        will be shaped `[batch_size, cell.state_size]`.  If it is a
+        `TensorShape`, this will be shaped `[batch_size] + cell.state_size`.
+        If it is a (possibly nested) tuple of ints or `TensorShape`, this will
+        be a tuple having the corresponding shapes. If cells are `LSTMCells`
+        `state` will be a tuple containing a `LSTMStateTuple` for each cell.
+
+    Raises:
+      TypeError: If `cell` is not an instance of RNNCell.
+      ValueError: If inputs is None or an empty list.
+    """
+    if not _like_rnncell(cell):
+        raise TypeError("cell must be an instance of RNNCell")
+
+    # By default, time_major==False and inputs are batch-major: shaped
+    #   [batch, time, depth]
+    # For internal calculations, we transpose to [time, batch, depth]
+    flat_input = nest.flatten(inputs)
+
+    if not time_major:
+        # (B,T,D) => (T,B,D)
+        flat_input = [ops.convert_to_tensor(input_) for input_ in flat_input]
+        flat_input = tuple(_transpose_batch_time(input_) for input_ in flat_input)
+
+    parallel_iterations = parallel_iterations or 32
+    if sequence_length is not None:
+        sequence_length = math_ops.to_int32(sequence_length)
+        if sequence_length.get_shape().ndims not in (None, 1):
+            raise ValueError(
+                "sequence_length must be a vector of length batch_size, "
+                "but saw shape: %s" % sequence_length.get_shape())
+        sequence_length = array_ops.identity(  # Just to find it in the graph.
+            sequence_length, name="sequence_length")
+
+    # Create a new scope in which the caching device is either
+    # determined by the parent scope, or is set to place the cached
+    # Variable using the same placement as for the rest of the RNN.
+    with vs.variable_scope(scope or "rnn") as varscope:
+        if varscope.caching_device is None:
+            varscope.set_caching_device(lambda op: op.device)
+        batch_size = _best_effort_input_batch_size(flat_input)
+
+        if initial_state is not None:
+            state = initial_state
+        else:
+            if not dtype:
+                raise ValueError("If there is no initial_state, you must give a dtype.")
+            state = cell.zero_state(batch_size, dtype)
+
+        def _assert_has_shape(x, shape):
+            x_shape = array_ops.shape(x)
+            packed_shape = array_ops.stack(shape)
+            return control_flow_ops.Assert(
+                math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)),
+                ["Expected shape for Tensor %s is " % x.name,
+                 packed_shape, " but saw shape: ", x_shape])
+
+        if sequence_length is not None:
+            # Perform some shape validation
+            with ops.control_dependencies(
+                    [_assert_has_shape(sequence_length, [batch_size])]):
+                sequence_length = array_ops.identity(
+                    sequence_length, name="CheckSeqLen")
+
+        inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input)
+
+        (outputs, final_state) = _dynamic_rnn_loop(
+            cell,
+            inputs,
+            state,
+            parallel_iterations=parallel_iterations,
+            swap_memory=swap_memory,
+            att_scores=att_scores,
+            sequence_length=sequence_length,
+            dtype=dtype)
+
+        # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].
+        # If we are performing batch-major calculations, transpose output back
+        # to shape [batch, time, depth]
+        if not time_major:
+            # (T,B,D) => (B,T,D)
+            outputs = nest.map_structure(_transpose_batch_time, outputs)
+
+        return (outputs, final_state)
+
+
+def _dynamic_rnn_loop(cell,
+                      inputs,
+                      initial_state,
+                      parallel_iterations,
+                      swap_memory,
+                      att_scores=None,
+                      sequence_length=None,
+                      dtype=None):
+    """Internal implementation of Dynamic RNN.
+
+    Args:
+      cell: An instance of RNNCell.
+      inputs: A `Tensor` of shape [time, batch_size, input_size], or a nested
+        tuple of such elements.
+      initial_state: A `Tensor` of shape `[batch_size, state_size]`, or if
+        `cell.state_size` is a tuple, then this should be a tuple of
+        tensors having shapes `[batch_size, s] for s in cell.state_size`.
+      parallel_iterations: Positive Python int.
+      swap_memory: A Python boolean
+      sequence_length: (optional) An `int32` `Tensor` of shape [batch_size].
+      dtype: (optional) Expected dtype of output. If not specified, inferred from
+        initial_state.
+
+    Returns:
+      Tuple `(final_outputs, final_state)`.
+      final_outputs:
+        A `Tensor` of shape `[time, batch_size, cell.output_size]`.  If
+        `cell.output_size` is a (possibly nested) tuple of ints or `TensorShape`
+        objects, then this returns a (possibly nsted) tuple of Tensors matching
+        the corresponding shapes.
+      final_state:
+        A `Tensor`, or possibly nested tuple of Tensors, matching in length
+        and shapes to `initial_state`.
+
+    Raises:
+      ValueError: If the input depth cannot be inferred via shape inference
+        from the inputs.
+    """
+    state = initial_state
+    assert isinstance(parallel_iterations, int), "parallel_iterations must be int"
+
+    state_size = cell.state_size
+
+    flat_input = nest.flatten(inputs)
+    flat_output_size = nest.flatten(cell.output_size)
+
+    # Construct an initial output
+    input_shape = array_ops.shape(flat_input[0])
+    time_steps = input_shape[0]
+    batch_size = _best_effort_input_batch_size(flat_input)
+
+    inputs_got_shape = tuple(input_.get_shape().with_rank_at_least(3)
+                             for input_ in flat_input)
+
+    const_time_steps, const_batch_size = inputs_got_shape[0].as_list()[:2]
+
+    for shape in inputs_got_shape:
+        if not shape[2:].is_fully_defined():
+            raise ValueError(
+                "Input size (depth of inputs) must be accessible via shape inference,"
+                " but saw value None.")
+        got_time_steps = shape[0].value
+        got_batch_size = shape[1].value
+        if const_time_steps != got_time_steps:
+            raise ValueError(
+                "Time steps is not the same for all the elements in the input in a "
+                "batch.")
+        if const_batch_size != got_batch_size:
+            raise ValueError(
+                "Batch_size is not the same for all the elements in the input.")
+
+    # Prepare dynamic conditional copying of state & output
+    def _create_zero_arrays(size):
+        size = _concat(batch_size, size)
+        return array_ops.zeros(
+            array_ops.stack(size), _infer_state_dtype(dtype, state))
+
+    flat_zero_output = tuple(_create_zero_arrays(output)
+                             for output in flat_output_size)
+    zero_output = nest.pack_sequence_as(structure=cell.output_size,
+                                        flat_sequence=flat_zero_output)
+
+    if sequence_length is not None:
+        min_sequence_length = math_ops.reduce_min(sequence_length)
+        max_sequence_length = math_ops.reduce_max(sequence_length)
+
+    time = array_ops.constant(0, dtype=dtypes.int32, name="time")
+
+    with ops.name_scope("dynamic_rnn") as scope:
+        base_name = scope
+
+    def _create_ta(name, dtype):
+        return tensor_array_ops.TensorArray(dtype=dtype,
+                                            size=time_steps,
+                                            tensor_array_name=base_name + name)
+
+    output_ta = tuple(_create_ta("output_%d" % i,
+                                 _infer_state_dtype(dtype, state))
+                      for i in range(len(flat_output_size)))
+    input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype)
+                     for i in range(len(flat_input)))
+
+    input_ta = tuple(ta.unstack(input_)
+                     for ta, input_ in zip(input_ta, flat_input))
+
+    def _time_step(time, output_ta_t, state, att_scores=None):
+        """Take a time step of the dynamic RNN.
+
+        Args:
+          time: int32 scalar Tensor.
+          output_ta_t: List of `TensorArray`s that represent the output.
+          state: nested tuple of vector tensors that represent the state.
+
+        Returns:
+          The tuple (time + 1, output_ta_t with updated flow, new_state).
+        """
+
+        input_t = tuple(ta.read(time) for ta in input_ta)
+        # Restore some shape information
+        for input_, shape in zip(input_t, inputs_got_shape):
+            input_.set_shape(shape[1:])
+
+        input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t)
+        if att_scores is not None:
+            att_score = att_scores[:, time, :]
+            call_cell = lambda: cell(input_t, state, att_score)
+        else:
+            call_cell = lambda: cell(input_t, state)
+
+        if sequence_length is not None:
+            (output, new_state) = _rnn_step(
+                time=time,
+                sequence_length=sequence_length,
+                min_sequence_length=min_sequence_length,
+                max_sequence_length=max_sequence_length,
+                zero_output=zero_output,
+                state=state,
+                call_cell=call_cell,
+                state_size=state_size,
+                skip_conditionals=True)
+        else:
+            (output, new_state) = call_cell()
+
+        # Pack state if using state tuples
+        output = nest.flatten(output)
+
+        output_ta_t = tuple(
+            ta.write(time, out) for ta, out in zip(output_ta_t, output))
+        if att_scores is not None:
+            return (time + 1, output_ta_t, new_state, att_scores)
+        else:
+            return (time + 1, output_ta_t, new_state)
+
+    if att_scores is not None:
+        _, output_final_ta, final_state, _ = control_flow_ops.while_loop(
+            cond=lambda time, *_: time < time_steps,
+            body=_time_step,
+            loop_vars=(time, output_ta, state, att_scores),
+            parallel_iterations=parallel_iterations,
+            swap_memory=swap_memory)
+    else:
+        _, output_final_ta, final_state = control_flow_ops.while_loop(
+            cond=lambda time, *_: time < time_steps,
+            body=_time_step,
+            loop_vars=(time, output_ta, state),
+            parallel_iterations=parallel_iterations,
+            swap_memory=swap_memory)
+
+    # Unpack final output if not using output tuples.
+    final_outputs = tuple(ta.stack() for ta in output_final_ta)
+
+    # Restore some shape information
+    for output, output_size in zip(final_outputs, flat_output_size):
+        shape = _concat(
+            [const_time_steps, const_batch_size], output_size, static=True)
+        output.set_shape(shape)
+
+    final_outputs = nest.pack_sequence_as(
+        structure=cell.output_size, flat_sequence=final_outputs)
+
+    return (final_outputs, final_state)
+
+
+def raw_rnn(cell, loop_fn,
+            parallel_iterations=None, swap_memory=False, scope=None):
+    """Creates an `RNN` specified by RNNCell `cell` and loop function `loop_fn`.
+
+    **NOTE: This method is still in testing, and the API may change.**
+
+    This function is a more primitive version of `dynamic_rnn` that provides
+    more direct access to the inputs each iteration.  It also provides more
+    control over when to start and finish reading the sequence, and
+    what to emit for the output.
+
+    For example, it can be used to implement the dynamic decoder of a seq2seq
+    model.
+
+    Instead of working with `Tensor` objects, most operations work with
+    `TensorArray` objects directly.
+
+    The operation of `raw_rnn`, in pseudo-code, is basically the following:
+
+    ```python
+    time = tf.constant(0, dtype=tf.int32)
+    (finished, next_input, initial_state, _, loop_state) = loop_fn(
+        time=time, cell_output=None, cell_state=None, loop_state=None)
+    emit_ta = TensorArray(dynamic_size=True, dtype=initial_state.dtype)
+    state = initial_state
+    while not all(finished):
+      (output, cell_state) = cell(next_input, state)
+      (next_finished, next_input, next_state, emit, loop_state) = loop_fn(
+          time=time + 1, cell_output=output, cell_state=cell_state,
+          loop_state=loop_state)
+      # Emit zeros and copy forward state for minibatch entries that are finished.
+      state = tf.where(finished, state, next_state)
+      emit = tf.where(finished, tf.zeros_like(emit), emit)
+      emit_ta = emit_ta.write(time, emit)
+      # If any new minibatch entries are marked as finished, mark these.
+      finished = tf.logical_or(finished, next_finished)
+      time += 1
+    return (emit_ta, state, loop_state)
+    ```
+
+    with the additional properties that output and state may be (possibly nested)
+    tuples, as determined by `cell.output_size` and `cell.state_size`, and
+    as a result the final `state` and `emit_ta` may themselves be tuples.
+
+    A simple implementation of `dynamic_rnn` via `raw_rnn` looks like this:
+
+    ```python
+    inputs = tf.placeholder(shape=(max_time, batch_size, input_depth),
+                            dtype=tf.float32)
+    sequence_length = tf.placeholder(shape=(batch_size,), dtype=tf.int32)
+    inputs_ta = tf.TensorArray(dtype=tf.float32, size=max_time)
+    inputs_ta = inputs_ta.unstack(inputs)
+
+    cell = tf.contrib.rnn.LSTMCell(num_units)
+
+    def loop_fn(time, cell_output, cell_state, loop_state):
+      emit_output = cell_output  # == None for time == 0
+      if cell_output is None:  # time == 0
+        next_cell_state = cell.zero_state(batch_size, tf.float32)
+      else:
+        next_cell_state = cell_state
+      elements_finished = (time >= sequence_length)
+      finished = tf.reduce_all(elements_finished)
+      next_input = tf.cond(
+          finished,
+          lambda: tf.zeros([batch_size, input_depth], dtype=tf.float32),
+          lambda: inputs_ta.read(time))
+      next_loop_state = None
+      return (elements_finished, next_input, next_cell_state,
+              emit_output, next_loop_state)
+
+    outputs_ta, final_state, _ = raw_rnn(cell, loop_fn)
+    outputs = outputs_ta.stack()
+    ```
+
+    Args:
+      cell: An instance of RNNCell.
+      loop_fn: A callable that takes inputs
+        `(time, cell_output, cell_state, loop_state)`
+        and returns the tuple
+        `(finished, next_input, next_cell_state, emit_output, next_loop_state)`.
+        Here `time` is an int32 scalar `Tensor`, `cell_output` is a
+        `Tensor` or (possibly nested) tuple of tensors as determined by
+        `cell.output_size`, and `cell_state` is a `Tensor`
+        or (possibly nested) tuple of tensors, as determined by the `loop_fn`
+        on its first call (and should match `cell.state_size`).
+        The outputs are: `finished`, a boolean `Tensor` of
+        shape `[batch_size]`, `next_input`: the next input to feed to `cell`,
+        `next_cell_state`: the next state to feed to `cell`,
+        and `emit_output`: the output to store for this iteration.
+
+        Note that `emit_output` should be a `Tensor` or (possibly nested)
+        tuple of tensors with shapes and structure matching `cell.output_size`
+        and `cell_output` above.  The parameter `cell_state` and output
+        `next_cell_state` may be either a single or (possibly nested) tuple
+        of tensors.  The parameter `loop_state` and
+        output `next_loop_state` may be either a single or (possibly nested) tuple
+        of `Tensor` and `TensorArray` objects.  This last parameter
+        may be ignored by `loop_fn` and the return value may be `None`.  If it
+        is not `None`, then the `loop_state` will be propagated through the RNN
+        loop, for use purely by `loop_fn` to keep track of its own state.
+        The `next_loop_state` parameter returned may be `None`.
+
+        The first call to `loop_fn` will be `time = 0`, `cell_output = None`,
+        `cell_state = None`, and `loop_state = None`.  For this call:
+        The `next_cell_state` value should be the value with which to initialize
+        the cell's state.  It may be a final state from a previous RNN or it
+        may be the output of `cell.zero_state()`.  It should be a
+        (possibly nested) tuple structure of tensors.
+        If `cell.state_size` is an integer, this must be
+        a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`.
+        If `cell.state_size` is a `TensorShape`, this must be a `Tensor` of
+        appropriate type and shape `[batch_size] + cell.state_size`.
+        If `cell.state_size` is a (possibly nested) tuple of ints or
+        `TensorShape`, this will be a tuple having the corresponding shapes.
+        The `emit_output` value may be either `None` or a (possibly nested)
+        tuple structure of tensors, e.g.,
+        `(tf.zeros(shape_0, dtype=dtype_0), tf.zeros(shape_1, dtype=dtype_1))`.
+        If this first `emit_output` return value is `None`,
+        then the `emit_ta` result of `raw_rnn` will have the same structure and
+        dtypes as `cell.output_size`.  Otherwise `emit_ta` will have the same
+        structure, shapes (prepended with a `batch_size` dimension), and dtypes
+        as `emit_output`.  The actual values returned for `emit_output` at this
+        initializing call are ignored.  Note, this emit structure must be
+        consistent across all time steps.
+
+      parallel_iterations: (Default: 32).  The number of iterations to run in
+        parallel.  Those operations which do not have any temporal dependency
+        and can be run in parallel, will be.  This parameter trades off
+        time for space.  Values >> 1 use more memory but take less time,
+        while smaller values use less memory but computations take longer.
+      swap_memory: Transparently swap the tensors produced in forward inference
+        but needed for back prop from GPU to CPU.  This allows training RNNs
+        which would typically not fit on a single GPU, with very minimal (or no)
+        performance penalty.
+      scope: VariableScope for the created subgraph; defaults to "rnn".
+
+    Returns:
+      A tuple `(emit_ta, final_state, final_loop_state)` where:
+
+      `emit_ta`: The RNN output `TensorArray`.
+         If `loop_fn` returns a (possibly nested) set of Tensors for
+         `emit_output` during initialization, (inputs `time = 0`,
+         `cell_output = None`, and `loop_state = None`), then `emit_ta` will
+         have the same structure, dtypes, and shapes as `emit_output` instead.
+         If `loop_fn` returns `emit_output = None` during this call,
+         the structure of `cell.output_size` is used:
+         If `cell.output_size` is a (possibly nested) tuple of integers
+         or `TensorShape` objects, then `emit_ta` will be a tuple having the
+         same structure as `cell.output_size`, containing TensorArrays whose
+         elements' shapes correspond to the shape data in `cell.output_size`.
+
+      `final_state`: The final cell state.  If `cell.state_size` is an int, this
+        will be shaped `[batch_size, cell.state_size]`.  If it is a
+        `TensorShape`, this will be shaped `[batch_size] + cell.state_size`.
+        If it is a (possibly nested) tuple of ints or `TensorShape`, this will
+        be a tuple having the corresponding shapes.
+
+      `final_loop_state`: The final loop state as returned by `loop_fn`.
+
+    Raises:
+      TypeError: If `cell` is not an instance of RNNCell, or `loop_fn` is not
+        a `callable`.
+    """
+
+    if not _like_rnncell(cell):
+        raise TypeError("cell must be an instance of RNNCell")
+    if not callable(loop_fn):
+        raise TypeError("loop_fn must be a callable")
+
+    parallel_iterations = parallel_iterations or 32
+
+    # Create a new scope in which the caching device is either
+    # determined by the parent scope, or is set to place the cached
+    # Variable using the same placement as for the rest of the RNN.
+    with vs.variable_scope(scope or "rnn") as varscope:
+        if varscope.caching_device is None:
+            varscope.set_caching_device(lambda op: op.device)
+
+        time = constant_op.constant(0, dtype=dtypes.int32)
+        (elements_finished, next_input, initial_state, emit_structure,
+         init_loop_state) = loop_fn(
+            time, None, None, None)  # time, cell_output, cell_state, loop_state
+        flat_input = nest.flatten(next_input)
+
+        # Need a surrogate loop state for the while_loop if none is available.
+        loop_state = (init_loop_state if init_loop_state is not None
+                      else constant_op.constant(0, dtype=dtypes.int32))
+
+        input_shape = [input_.get_shape() for input_ in flat_input]
+        static_batch_size = input_shape[0][0]
+
+        for input_shape_i in input_shape:
+            # Static verification that batch sizes all match
+            static_batch_size.merge_with(input_shape_i[0])
+
+        batch_size = static_batch_size.value
+        if batch_size is None:
+            batch_size = array_ops.shape(flat_input[0])[0]
+
+        nest.assert_same_structure(initial_state, cell.state_size)
+        state = initial_state
+        flat_state = nest.flatten(state)
+        flat_state = [ops.convert_to_tensor(s) for s in flat_state]
+        state = nest.pack_sequence_as(structure=state,
+                                      flat_sequence=flat_state)
+
+        if emit_structure is not None:
+            flat_emit_structure = nest.flatten(emit_structure)
+            flat_emit_size = [emit.shape if emit.shape.is_fully_defined() else
+                              array_ops.shape(emit) for emit in flat_emit_structure]
+            flat_emit_dtypes = [emit.dtype for emit in flat_emit_structure]
+        else:
+            emit_structure = cell.output_size
+            flat_emit_size = nest.flatten(emit_structure)
+            flat_emit_dtypes = [flat_state[0].dtype] * len(flat_emit_size)
+
+        flat_emit_ta = [
+            tensor_array_ops.TensorArray(
+                dtype=dtype_i, dynamic_size=True, size=0, name="rnn_output_%d" % i)
+            for i, dtype_i in enumerate(flat_emit_dtypes)]
+        emit_ta = nest.pack_sequence_as(structure=emit_structure,
+                                        flat_sequence=flat_emit_ta)
+        flat_zero_emit = [
+            array_ops.zeros(_concat(batch_size, size_i), dtype_i)
+            for size_i, dtype_i in zip(flat_emit_size, flat_emit_dtypes)]
+        zero_emit = nest.pack_sequence_as(structure=emit_structure,
+                                          flat_sequence=flat_zero_emit)
+
+        def condition(unused_time, elements_finished, *_):
+            return math_ops.logical_not(math_ops.reduce_all(elements_finished))
+
+        def body(time, elements_finished, current_input,
+                 emit_ta, state, loop_state):
+            """Internal while loop body for raw_rnn.
+
+            Args:
+              time: time scalar.
+              elements_finished: batch-size vector.
+              current_input: possibly nested tuple of input tensors.
+              emit_ta: possibly nested tuple of output TensorArrays.
+              state: possibly nested tuple of state tensors.
+              loop_state: possibly nested tuple of loop state tensors.
+
+            Returns:
+              Tuple having the same size as Args but with updated values.
+            """
+            (next_output, cell_state) = cell(current_input, state)
+
+            nest.assert_same_structure(state, cell_state)
+            nest.assert_same_structure(cell.output_size, next_output)
+
+            next_time = time + 1
+            (next_finished, next_input, next_state, emit_output,
+             next_loop_state) = loop_fn(
+                next_time, next_output, cell_state, loop_state)
+
+            nest.assert_same_structure(state, next_state)
+            nest.assert_same_structure(current_input, next_input)
+            nest.assert_same_structure(emit_ta, emit_output)
+
+            # If loop_fn returns None for next_loop_state, just reuse the
+            # previous one.
+            loop_state = loop_state if next_loop_state is None else next_loop_state
+
+            def _copy_some_through(current, candidate):
+                """Copy some tensors through via array_ops.where."""
+
+                def copy_fn(cur_i, cand_i):
+                    with ops.colocate_with(cand_i):
+                        return array_ops.where(elements_finished, cur_i, cand_i)
+
+                return nest.map_structure(copy_fn, current, candidate)
+
+            emit_output = _copy_some_through(zero_emit, emit_output)
+            next_state = _copy_some_through(state, next_state)
+
+            emit_ta = nest.map_structure(
+                lambda ta, emit: ta.write(time, emit), emit_ta, emit_output)
+
+            elements_finished = math_ops.logical_or(elements_finished, next_finished)
+
+            return (next_time, elements_finished, next_input,
+                    emit_ta, next_state, loop_state)
+
+        returned = control_flow_ops.while_loop(
+            condition, body, loop_vars=[
+                time, elements_finished, next_input,
+                emit_ta, state, loop_state],
+            parallel_iterations=parallel_iterations,
+            swap_memory=swap_memory)
+
+        (emit_ta, final_state, final_loop_state) = returned[-3:]
+
+        if init_loop_state is None:
+            final_loop_state = None
+
+        return (emit_ta, final_state, final_loop_state)
+
+
+def static_rnn(cell,
+               inputs,
+               initial_state=None,
+               dtype=None,
+               sequence_length=None,
+               scope=None):
+    """Creates a recurrent neural network specified by RNNCell `cell`.
+
+    The simplest form of RNN network generated is:
+
+    ```python
+      state = cell.zero_state(...)
+      outputs = []
+      for input_ in inputs:
+        output, state = cell(input_, state)
+        outputs.append(output)
+      return (outputs, state)
+    ```
+    However, a few other options are available:
+
+    An initial state can be provided.
+    If the sequence_length vector is provided, dynamic calculation is performed.
+    This method of calculation does not compute the RNN steps past the maximum
+    sequence length of the minibatch (thus saving computational time),
+    and properly propagates the state at an example's sequence length
+    to the final state output.
+
+    The dynamic calculation performed is, at time `t` for batch row `b`,
+
+    ```python
+      (output, state)(b, t) =
+        (t >= sequence_length(b))
+          ? (zeros(cell.output_size), states(b, sequence_length(b) - 1))
+          : cell(input(b, t), state(b, t - 1))
+    ```
+
+    Args:
+      cell: An instance of RNNCell.
+      inputs: A length T list of inputs, each a `Tensor` of shape
+        `[batch_size, input_size]`, or a nested tuple of such elements.
+      initial_state: (optional) An initial state for the RNN.
+        If `cell.state_size` is an integer, this must be
+        a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`.
+        If `cell.state_size` is a tuple, this should be a tuple of
+        tensors having shapes `[batch_size, s] for s in cell.state_size`.
+      dtype: (optional) The data type for the initial state and expected output.
+        Required if initial_state is not provided or RNN state has a heterogeneous
+        dtype.
+      sequence_length: Specifies the length of each sequence in inputs.
+        An int32 or int64 vector (tensor) size `[batch_size]`, values in `[0, T)`.
+      scope: VariableScope for the created subgraph; defaults to "rnn".
+
+    Returns:
+      A pair (outputs, state) where:
+
+      - outputs is a length T list of outputs (one for each input), or a nested
+        tuple of such elements.
+      - state is the final state
+
+    Raises:
+      TypeError: If `cell` is not an instance of RNNCell.
+      ValueError: If `inputs` is `None` or an empty list, or if the input depth
+        (column size) cannot be inferred from inputs via shape inference.
+    """
+
+    if not _like_rnncell(cell):
+        raise TypeError("cell must be an instance of RNNCell")
+    if not nest.is_sequence(inputs):
+        raise TypeError("inputs must be a sequence")
+    if not inputs:
+        raise ValueError("inputs must not be empty")
+
+    outputs = []
+    # Create a new scope in which the caching device is either
+    # determined by the parent scope, or is set to place the cached
+    # Variable using the same placement as for the rest of the RNN.
+    with vs.variable_scope(scope or "rnn") as varscope:
+        if varscope.caching_device is None:
+            varscope.set_caching_device(lambda op: op.device)
+
+        # Obtain the first sequence of the input
+        first_input = inputs
+        while nest.is_sequence(first_input):
+            first_input = first_input[0]
+
+        # Temporarily avoid EmbeddingWrapper and seq2seq badness
+        # TODO(lukaszkaiser): remove EmbeddingWrapper
+        if first_input.get_shape().ndims != 1:
+
+            input_shape = first_input.get_shape().with_rank_at_least(2)
+            fixed_batch_size = input_shape[0]
+
+            flat_inputs = nest.flatten(inputs)
+            for flat_input in flat_inputs:
+                input_shape = flat_input.get_shape().with_rank_at_least(2)
+                batch_size, input_size = input_shape[0], input_shape[1:]
+                fixed_batch_size.merge_with(batch_size)
+                for i, size in enumerate(input_size):
+                    if size.value is None:
+                        raise ValueError(
+                            "Input size (dimension %d of inputs) must be accessible via "
+                            "shape inference, but saw value None." % i)
+        else:
+            fixed_batch_size = first_input.get_shape().with_rank_at_least(1)[0]
+
+        if fixed_batch_size.value:
+            batch_size = fixed_batch_size.value
+        else:
+            batch_size = array_ops.shape(first_input)[0]
+        if initial_state is not None:
+            state = initial_state
+        else:
+            if not dtype:
+                raise ValueError("If no initial_state is provided, "
+                                 "dtype must be specified")
+            state = cell.zero_state(batch_size, dtype)
+
+        if sequence_length is not None:  # Prepare variables
+            sequence_length = ops.convert_to_tensor(
+                sequence_length, name="sequence_length")
+            if sequence_length.get_shape().ndims not in (None, 1):
+                raise ValueError(
+                    "sequence_length must be a vector of length batch_size")
+
+            def _create_zero_output(output_size):
+                # convert int to TensorShape if necessary
+                size = _concat(batch_size, output_size)
+                output = array_ops.zeros(
+                    array_ops.stack(size), _infer_state_dtype(dtype, state))
+                shape = _concat(fixed_batch_size.value, output_size, static=True)
+                output.set_shape(tensor_shape.TensorShape(shape))
+                return output
+
+            output_size = cell.output_size
+            flat_output_size = nest.flatten(output_size)
+            flat_zero_output = tuple(
+                _create_zero_output(size) for size in flat_output_size)
+            zero_output = nest.pack_sequence_as(
+                structure=output_size, flat_sequence=flat_zero_output)
+
+            sequence_length = math_ops.to_int32(sequence_length)
+            min_sequence_length = math_ops.reduce_min(sequence_length)
+            max_sequence_length = math_ops.reduce_max(sequence_length)
+
+        for time, input_ in enumerate(inputs):
+            if time > 0:
+                varscope.reuse_variables()
+            # pylint: disable=cell-var-from-loop
+            call_cell = lambda: cell(input_, state)
+            # pylint: enable=cell-var-from-loop
+            if sequence_length is not None:
+                (output, state) = _rnn_step(
+                    time=time,
+                    sequence_length=sequence_length,
+                    min_sequence_length=min_sequence_length,
+                    max_sequence_length=max_sequence_length,
+                    zero_output=zero_output,
+                    state=state,
+                    call_cell=call_cell,
+                    state_size=cell.state_size)
+            else:
+                (output, state) = call_cell()
+
+            outputs.append(output)
+
+        return (outputs, state)
+
+
+def static_state_saving_rnn(cell,
+                            inputs,
+                            state_saver,
+                            state_name,
+                            sequence_length=None,
+                            scope=None):
+    """RNN that accepts a state saver for time-truncated RNN calculation.
+
+    Args:
+      cell: An instance of `RNNCell`.
+      inputs: A length T list of inputs, each a `Tensor` of shape
+        `[batch_size, input_size]`.
+      state_saver: A state saver object with methods `state` and `save_state`.
+      state_name: Python string or tuple of strings.  The name to use with the
+        state_saver. If the cell returns tuples of states (i.e.,
+        `cell.state_size` is a tuple) then `state_name` should be a tuple of
+        strings having the same length as `cell.state_size`.  Otherwise it should
+        be a single string.
+      sequence_length: (optional) An int32/int64 vector size [batch_size].
+        See the documentation for rnn() for more details about sequence_length.
+      scope: VariableScope for the created subgraph; defaults to "rnn".
+
+    Returns:
+      A pair (outputs, state) where:
+        outputs is a length T list of outputs (one for each input)
+        states is the final state
+
+    Raises:
+      TypeError: If `cell` is not an instance of RNNCell.
+      ValueError: If `inputs` is `None` or an empty list, or if the arity and
+       type of `state_name` does not match that of `cell.state_size`.
+    """
+    state_size = cell.state_size
+    state_is_tuple = nest.is_sequence(state_size)
+    state_name_tuple = nest.is_sequence(state_name)
+
+    if state_is_tuple != state_name_tuple:
+        raise ValueError("state_name should be the same type as cell.state_size.  "
+                         "state_name: %s, cell.state_size: %s" % (str(state_name),
+                                                                  str(state_size)))
+
+    if state_is_tuple:
+        state_name_flat = nest.flatten(state_name)
+        state_size_flat = nest.flatten(state_size)
+
+        if len(state_name_flat) != len(state_size_flat):
+            raise ValueError("#elems(state_name) != #elems(state_size): %d vs. %d" %
+                             (len(state_name_flat), len(state_size_flat)))
+
+        initial_state = nest.pack_sequence_as(
+            structure=state_size,
+            flat_sequence=[state_saver.state(s) for s in state_name_flat])
+    else:
+        initial_state = state_saver.state(state_name)
+
+    (outputs, state) = static_rnn(
+        cell,
+        inputs,
+        initial_state=initial_state,
+        sequence_length=sequence_length,
+        scope=scope)
+
+    if state_is_tuple:
+        flat_state = nest.flatten(state)
+        state_name = nest.flatten(state_name)
+        save_state = [
+            state_saver.save_state(name, substate)
+            for name, substate in zip(state_name, flat_state)
+        ]
+    else:
+        save_state = [state_saver.save_state(state_name, state)]
+
+    with ops.control_dependencies(save_state):
+        last_output = outputs[-1]
+        flat_last_output = nest.flatten(last_output)
+        flat_last_output = [
+            array_ops.identity(output) for output in flat_last_output
+        ]
+        outputs[-1] = nest.pack_sequence_as(
+            structure=last_output, flat_sequence=flat_last_output)
+
+    return (outputs, state)
+
+
+def static_bidirectional_rnn(cell_fw,
+                             cell_bw,
+                             inputs,
+                             initial_state_fw=None,
+                             initial_state_bw=None,
+                             dtype=None,
+                             sequence_length=None,
+                             scope=None):
+    """Creates a bidirectional recurrent neural network.
+
+    Similar to the unidirectional case above (rnn) but takes input and builds
+    independent forward and backward RNNs with the final forward and backward
+    outputs depth-concatenated, such that the output will have the format
+    [time][batch][cell_fw.output_size + cell_bw.output_size]. The input_size of
+    forward and backward cell must match. The initial state for both directions
+    is zero by default (but can be set optionally) and no intermediate states are
+    ever returned -- the network is fully unrolled for the given (passed in)
+    length(s) of the sequence(s) or completely unrolled if length(s) is not given.
+
+    Args:
+      cell_fw: An instance of RNNCell, to be used for forward direction.
+      cell_bw: An instance of RNNCell, to be used for backward direction.
+      inputs: A length T list of inputs, each a tensor of shape
+        [batch_size, input_size], or a nested tuple of such elements.
+      initial_state_fw: (optional) An initial state for the forward RNN.
+        This must be a tensor of appropriate type and shape
+        `[batch_size, cell_fw.state_size]`.
+        If `cell_fw.state_size` is a tuple, this should be a tuple of
+        tensors having shapes `[batch_size, s] for s in cell_fw.state_size`.
+      initial_state_bw: (optional) Same as for `initial_state_fw`, but using
+        the corresponding properties of `cell_bw`.
+      dtype: (optional) The data type for the initial state.  Required if
+        either of the initial states are not provided.
+      sequence_length: (optional) An int32/int64 vector, size `[batch_size]`,
+        containing the actual lengths for each of the sequences.
+      scope: VariableScope for the created subgraph; defaults to
+        "bidirectional_rnn"
+
+    Returns:
+      A tuple (outputs, output_state_fw, output_state_bw) where:
+        outputs is a length `T` list of outputs (one for each input), which
+          are depth-concatenated forward and backward outputs.
+        output_state_fw is the final state of the forward rnn.
+        output_state_bw is the final state of the backward rnn.
+
+    Raises:
+      TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`.
+      ValueError: If inputs is None or an empty list.
+    """
+
+    if not _like_rnncell(cell_fw):
+        raise TypeError("cell_fw must be an instance of RNNCell")
+    if not _like_rnncell(cell_bw):
+        raise TypeError("cell_bw must be an instance of RNNCell")
+    if not nest.is_sequence(inputs):
+        raise TypeError("inputs must be a sequence")
+    if not inputs:
+        raise ValueError("inputs must not be empty")
+
+    with vs.variable_scope(scope or "bidirectional_rnn"):
+        # Forward direction
+        with vs.variable_scope("fw") as fw_scope:
+            output_fw, output_state_fw = static_rnn(
+                cell_fw,
+                inputs,
+                initial_state_fw,
+                dtype,
+                sequence_length,
+                scope=fw_scope)
+
+        # Backward direction
+        with vs.variable_scope("bw") as bw_scope:
+            reversed_inputs = _reverse_seq(inputs, sequence_length)
+            tmp, output_state_bw = static_rnn(
+                cell_bw,
+                reversed_inputs,
+                initial_state_bw,
+                dtype,
+                sequence_length,
+                scope=bw_scope)
+
+    output_bw = _reverse_seq(tmp, sequence_length)
+    # Concat each of the forward/backward outputs
+    flat_output_fw = nest.flatten(output_fw)
+    flat_output_bw = nest.flatten(output_bw)
+
+    flat_outputs = tuple(
+        array_ops.concat([fw, bw], 1)
+        for fw, bw in zip(flat_output_fw, flat_output_bw))
+
+    outputs = nest.pack_sequence_as(
+        structure=output_fw, flat_sequence=flat_outputs)
+
+    return (outputs, output_state_fw, output_state_bw)
diff --git a/recommendation/Basic-DIEN-Demo/source_code/Dice.py b/recommendation/Basic-DIEN-Demo/source_code/Dice.py
new file mode 100755
index 00000000..160fb3d9
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/Dice.py
@@ -0,0 +1,35 @@
+import tensorflow as tf
+
+def dice(_x, axis=-1, epsilon=0.000000001, name=''):
+  with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
+    alphas = tf.get_variable('alpha'+name, _x.get_shape()[-1],
+                         initializer=tf.constant_initializer(0.0),
+                         dtype=tf.float32)
+    input_shape = list(_x.get_shape())
+
+    reduction_axes = list(range(len(input_shape)))
+    del reduction_axes[axis]
+    broadcast_shape = [1] * len(input_shape)
+    broadcast_shape[axis] = input_shape[axis]
+
+  # case: train mode (uses stats of the current batch)
+  mean = tf.reduce_mean(_x, axis=reduction_axes)
+  brodcast_mean = tf.reshape(mean, broadcast_shape)
+  std = tf.reduce_mean(tf.square(_x - brodcast_mean) + epsilon, axis=reduction_axes)
+  std = tf.sqrt(std)
+  brodcast_std = tf.reshape(std, broadcast_shape)
+  x_normed = (_x - brodcast_mean) / (brodcast_std + epsilon)
+  # x_normed = tf.layers.batch_normalization(_x, center=False, scale=False)
+  x_p = tf.sigmoid(x_normed)
+
+
+  return alphas * (1.0 - x_p) * _x + x_p * _x
+
+def parametric_relu(_x):
+  alphas = tf.get_variable('alpha', _x.get_shape()[-1],
+                       initializer=tf.constant_initializer(0.0),
+                       dtype=tf.float32)
+  pos = tf.nn.relu(_x)
+  neg = alphas * (_x - abs(_x)) * 0.5
+
+  return pos + neg
diff --git a/recommendation/Basic-DIEN-Demo/source_code/data_iterator.py b/recommendation/Basic-DIEN-Demo/source_code/data_iterator.py
new file mode 100755
index 00000000..c4ac4bb0
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/data_iterator.py
@@ -0,0 +1,210 @@
+import numpy
+import json
+import pickle as pkl
+import random
+
+import gzip
+
+import shuffle
+
+def unicode_to_utf8(d):
+    return dict((key.encode("UTF-8"), value) for (key,value) in d.items())
+
+def load_dict(filename):
+    try:
+        with open(filename, 'rb') as f:
+            return unicode_to_utf8(json.load(f))
+    except:
+        with open(filename, 'rb') as f:
+            return unicode_to_utf8(pkl.load(f))
+
+
+def fopen(filename, mode='r'):
+    if filename.endswith('.gz'):
+        return gzip.open(filename, mode)
+    return open(filename, mode)
+
+
+class DataIterator:
+
+    def __init__(self, source,
+                 uid_voc,
+                 mid_voc,
+                 cat_voc,
+                 batch_size=128,
+                 maxlen=100,
+                 skip_empty=False,
+                 shuffle_each_epoch=False,
+                 sort_by_length=True,
+                 max_batch_size=20,
+                 minlen=None):
+        if shuffle_each_epoch:
+            self.source_orig = source
+            self.source = shuffle.main(self.source_orig, temporary=True)
+        else:
+            self.source = fopen(source, 'r')
+        self.source_dicts = []
+        for source_dict in [uid_voc, mid_voc, cat_voc]:
+            self.source_dicts.append(load_dict(source_dict))
+
+        f_meta = open("/Users/meituan_sxw/Downloads/dien-master/item-info", "r")
+        meta_map = {}
+        for line in f_meta:
+            arr = line.strip().split("\t")
+            if arr[0] not in meta_map:
+                meta_map[arr[0]] = arr[1]
+        self.meta_id_map ={}
+        for key in meta_map:
+            val = meta_map[key]
+            if key in self.source_dicts[1]:
+                mid_idx = self.source_dicts[1][key]
+            else:
+                mid_idx = 0
+            if val in self.source_dicts[2]:
+                cat_idx = self.source_dicts[2][val]
+            else:
+                cat_idx = 0
+            self.meta_id_map[mid_idx] = cat_idx # 得到movieid对应的catid
+
+
+        f_review = open("/Users/meituan_sxw/Downloads/dien-master/reviews-info", "r")
+        self.mid_list_for_random = []
+        for line in f_review:
+            arr = line.strip().split("\t")
+            tmp_idx = 0
+            if arr[1] in self.source_dicts[1]:
+                tmp_idx = self.source_dicts[1][arr[1]]
+            self.mid_list_for_random.append(tmp_idx) #用户的推荐历史
+
+        self.batch_size = batch_size
+        self.maxlen = maxlen
+        self.minlen = minlen
+        self.skip_empty = skip_empty
+
+        self.n_uid = len(self.source_dicts[0])
+        self.n_mid = len(self.source_dicts[1])
+        self.n_cat = len(self.source_dicts[2])
+
+        self.shuffle = shuffle_each_epoch
+        self.sort_by_length = sort_by_length
+
+        self.source_buffer = []
+        self.k = batch_size * max_batch_size
+
+        self.end_of_data = False
+
+    def get_n(self):
+        return self.n_uid, self.n_mid, self.n_cat
+
+    def __iter__(self):
+        return self
+
+    def reset(self):
+        if self.shuffle:
+            self.source= shuffle.main(self.source_orig, temporary=True)
+        else:
+            self.source.seek(0)
+
+    def next(self):
+        if self.end_of_data:
+            self.end_of_data = False
+            self.reset()
+            raise StopIteration
+
+        source = []
+        target = []
+
+        if len(self.source_buffer) == 0:
+            for k_ in range(self.k):
+                ss = self.source.readline()
+                if ss == "":
+                    break
+                self.source_buffer.append(ss.strip("\n").split("\t"))
+
+            # sort by  history behavior length
+            if self.sort_by_length:
+                his_length = numpy.array([len(s[4].split("")) for s in self.source_buffer])
+                tidx = his_length.argsort()
+
+                _sbuf = [self.source_buffer[i] for i in tidx]
+                self.source_buffer = _sbuf
+            else:
+                self.source_buffer.reverse()
+
+        if len(self.source_buffer) == 0:
+            self.end_of_data = False
+            self.reset()
+            raise StopIteration
+
+        try:
+
+            # actual work here
+            while True:
+
+                # read from source file and map to word index
+                try:
+                    ss = self.source_buffer.pop()
+                except IndexError:
+                    break
+
+                uid = self.source_dicts[0][ss[1]] if ss[1] in self.source_dicts[0] else 0
+                mid = self.source_dicts[1][ss[2]] if ss[2] in self.source_dicts[1] else 0
+                cat = self.source_dicts[2][ss[3]] if ss[3] in self.source_dicts[2] else 0
+                tmp = []
+                for fea in ss[4].split(""):
+                    m = self.source_dicts[1][fea] if fea in self.source_dicts[1] else 0
+                    tmp.append(m)
+                mid_list = tmp
+
+                tmp1 = []
+                for fea in ss[5].split(""):
+                    c = self.source_dicts[2][fea] if fea in self.source_dicts[2] else 0
+                    tmp1.append(c)
+                cat_list = tmp1
+
+                # read from source file and map to word index
+
+                #if len(mid_list) > self.maxlen:
+                #    continue
+                if self.minlen != None:
+                    if len(mid_list) >= self.minlen:
+                        continue
+                if self.skip_empty and (not mid_list):
+                    continue
+
+                noclk_mid_list = []
+                noclk_cat_list = []
+                for pos_mid in mid_list:
+                    noclk_tmp_mid = []
+                    noclk_tmp_cat = []
+                    noclk_index = 0
+                    while True:
+                        noclk_mid_indx = random.randint(0, len(self.mid_list_for_random)-1)
+                        noclk_mid = self.mid_list_for_random[noclk_mid_indx]
+                        if noclk_mid == pos_mid:
+                            continue
+                        noclk_tmp_mid.append(noclk_mid)
+                        noclk_tmp_cat.append(self.meta_id_map[noclk_mid])
+                        noclk_index += 1
+                        if noclk_index >= 5:
+                            break
+                    noclk_mid_list.append(noclk_tmp_mid)
+                    noclk_cat_list.append(noclk_tmp_cat)
+                print([uid, mid, cat, mid_list, cat_list, noclk_mid_list, noclk_cat_list])
+                source.append([uid, mid, cat, mid_list, cat_list, noclk_mid_list, noclk_cat_list])
+                target.append([float(ss[0]), 1-float(ss[0])])
+
+                if len(source) >= self.batch_size or len(target) >= self.batch_size:
+                    break
+        except IOError:
+            self.end_of_data = True
+
+        # all sentence pairs in maxibatch filtered out because of length
+        if len(source) == 0 or len(target) == 0:
+            source, target = self.next()
+
+
+
+        return source, target
+
+
diff --git a/recommendation/Basic-DIEN-Demo/source_code/generate_voc.py b/recommendation/Basic-DIEN-Demo/source_code/generate_voc.py
new file mode 100755
index 00000000..41170814
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/generate_voc.py
@@ -0,0 +1,65 @@
+import cPickle
+
+f_train = open("local_train_splitByUser", "r")
+uid_dict = {}
+mid_dict = {}
+cat_dict = {}
+
+iddd = 0
+for line in f_train:
+    arr = line.strip("\n").split("\t")
+    clk = arr[0]
+    uid = arr[1]
+    mid = arr[2]
+    cat = arr[3]
+    mid_list = arr[4]
+    cat_list = arr[5]
+    if uid not in uid_dict:
+        uid_dict[uid] = 0
+    uid_dict[uid] += 1
+    if mid not in mid_dict:
+        mid_dict[mid] = 0
+    mid_dict[mid] += 1
+    if cat not in cat_dict:
+        cat_dict[cat] = 0
+    cat_dict[cat] += 1
+    if len(mid_list) == 0:
+        continue
+    for m in mid_list.split(""):
+        if m not in mid_dict:
+            mid_dict[m] = 0
+        mid_dict[m] += 1
+    #print iddd
+    iddd+=1
+    for c in cat_list.split(""):
+        if c not in cat_dict:
+            cat_dict[c] = 0
+        cat_dict[c] += 1
+
+sorted_uid_dict = sorted(uid_dict.iteritems(), key=lambda x:x[1], reverse=True)
+sorted_mid_dict = sorted(mid_dict.iteritems(), key=lambda x:x[1], reverse=True)
+sorted_cat_dict = sorted(cat_dict.iteritems(), key=lambda x:x[1], reverse=True)
+
+uid_voc = {}
+index = 0
+for key, value in sorted_uid_dict:
+    uid_voc[key] = index
+    index += 1
+
+mid_voc = {}
+mid_voc["default_mid"] = 0
+index = 1
+for key, value in sorted_mid_dict:
+    mid_voc[key] = index
+    index += 1
+
+cat_voc = {}
+cat_voc["default_cat"] = 0
+index = 1
+for key, value in sorted_cat_dict:
+    cat_voc[key] = index
+    index += 1
+
+cPickle.dump(uid_voc, open("uid_voc.pkl", "w"))
+cPickle.dump(mid_voc, open("mid_voc.pkl", "w"))
+cPickle.dump(cat_voc, open("cat_voc.pkl", "w"))
diff --git a/recommendation/Basic-DIEN-Demo/source_code/local_aggretor.py b/recommendation/Basic-DIEN-Demo/source_code/local_aggretor.py
new file mode 100755
index 00000000..86a0163b
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/local_aggretor.py
@@ -0,0 +1,45 @@
+import sys
+import hashlib
+import random
+
+fin = open("jointed-new-split-info", "r")
+ftrain = open("local_train", "w")
+ftest = open("local_test", "w")
+
+last_user = "0"
+common_fea = ""
+line_idx = 0
+for line in fin:
+    items = line.strip().split("\t")
+    ds = items[0]
+    clk = int(items[1])
+    user = items[2]
+    movie_id = items[3]
+    dt = items[5]
+    cat1 = items[6]
+
+    if ds=="20180118":
+        fo = ftrain
+    else:
+        fo = ftest
+    if user != last_user:
+        movie_id_list = []
+        cate1_list = []
+        #print >> fo, items[1] + "\t" + user + "\t" + movie_id + "\t" + cat1 +"\t" + "" + "\t" + "" 
+    else:
+        history_clk_num = len(movie_id_list)
+        cat_str = ""
+        mid_str = ""
+        for c1 in cate1_list:
+            cat_str += c1 + ""
+        for mid in movie_id_list:
+            mid_str += mid + ""
+        if len(cat_str) > 0: cat_str = cat_str[:-1]
+        if len(mid_str) > 0: mid_str = mid_str[:-1]
+        if history_clk_num >= 1:    # 8 is the average length of user behavior
+            print >> fo, items[1] + "\t" + user + "\t" + movie_id + "\t" + cat1 +"\t" + mid_str + "\t" + cat_str
+    last_user = user
+    if clk:
+        movie_id_list.append(movie_id)
+        cate1_list.append(cat1)                
+    line_idx += 1
diff --git a/recommendation/Basic-DIEN-Demo/source_code/model.py b/recommendation/Basic-DIEN-Demo/source_code/model.py
new file mode 100755
index 00000000..3640e8ea
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/model.py
@@ -0,0 +1,399 @@
+import tensorflow as tf
+from tensorflow.python.ops.rnn_cell import GRUCell
+from tensorflow.python.ops.rnn_cell import LSTMCell
+from tensorflow.python.ops.rnn import bidirectional_dynamic_rnn as bi_rnn
+#from tensorflow.python.ops.rnn import dynamic_rnn
+from rnn import dynamic_rnn
+from utils import *
+from Dice import dice
+
+class Model(object):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling = False):
+        with tf.name_scope('Inputs'):
+            self.mid_his_batch_ph = tf.placeholder(tf.int32, [None, None], name='mid_his_batch_ph')
+            self.cat_his_batch_ph = tf.placeholder(tf.int32, [None, None], name='cat_his_batch_ph')
+            self.uid_batch_ph = tf.placeholder(tf.int32, [None, ], name='uid_batch_ph')
+            self.mid_batch_ph = tf.placeholder(tf.int32, [None, ], name='mid_batch_ph')
+            self.cat_batch_ph = tf.placeholder(tf.int32, [None, ], name='cat_batch_ph')
+            self.mask = tf.placeholder(tf.float32, [None, None], name='mask')
+            self.seq_len_ph = tf.placeholder(tf.int32, [None], name='seq_len_ph')
+            self.target_ph = tf.placeholder(tf.float32, [None, None], name='target_ph')
+            self.lr = tf.placeholder(tf.float64, [])
+            self.use_negsampling =use_negsampling
+            if use_negsampling:
+                self.noclk_mid_batch_ph = tf.placeholder(tf.int32, [None, None, None], name='noclk_mid_batch_ph')
+                self.noclk_cat_batch_ph = tf.placeholder(tf.int32, [None, None, None], name='noclk_cat_batch_ph')
+
+        # Embedding layer
+        with tf.name_scope('Embedding_layer'):
+            self.uid_embeddings_var = tf.get_variable("uid_embedding_var", [n_uid, EMBEDDING_DIM])
+            tf.summary.histogram('uid_embeddings_var', self.uid_embeddings_var)
+            self.uid_batch_embedded = tf.nn.embedding_lookup(self.uid_embeddings_var, self.uid_batch_ph)
+
+            self.mid_embeddings_var = tf.get_variable("mid_embedding_var", [n_mid, EMBEDDING_DIM])
+            tf.summary.histogram('mid_embeddings_var', self.mid_embeddings_var)
+            self.mid_batch_embedded = tf.nn.embedding_lookup(self.mid_embeddings_var, self.mid_batch_ph)
+            self.mid_his_batch_embedded = tf.nn.embedding_lookup(self.mid_embeddings_var, self.mid_his_batch_ph)
+            if self.use_negsampling:
+                self.noclk_mid_his_batch_embedded = tf.nn.embedding_lookup(self.mid_embeddings_var, self.noclk_mid_batch_ph)
+
+            self.cat_embeddings_var = tf.get_variable("cat_embedding_var", [n_cat, EMBEDDING_DIM])
+            tf.summary.histogram('cat_embeddings_var', self.cat_embeddings_var)
+            self.cat_batch_embedded = tf.nn.embedding_lookup(self.cat_embeddings_var, self.cat_batch_ph)
+            self.cat_his_batch_embedded = tf.nn.embedding_lookup(self.cat_embeddings_var, self.cat_his_batch_ph)
+            if self.use_negsampling:
+                self.noclk_cat_his_batch_embedded = tf.nn.embedding_lookup(self.cat_embeddings_var, self.noclk_cat_batch_ph)
+
+        self.item_eb = tf.concat([self.mid_batch_embedded, self.cat_batch_embedded], 1)
+        self.item_his_eb = tf.concat([self.mid_his_batch_embedded, self.cat_his_batch_embedded], 2)
+        self.item_his_eb_sum = tf.reduce_sum(self.item_his_eb, 1)
+        if self.use_negsampling:
+            self.noclk_item_his_eb = tf.concat(
+                [self.noclk_mid_his_batch_embedded[:, :, 0, :], self.noclk_cat_his_batch_embedded[:, :, 0, :]], -1)
+            self.noclk_item_his_eb = tf.reshape(self.noclk_item_his_eb,
+                                                [-1, tf.shape(self.noclk_mid_his_batch_embedded)[1], 36])
+
+            self.noclk_his_eb = tf.concat([self.noclk_mid_his_batch_embedded, self.noclk_cat_his_batch_embedded], -1)
+            self.noclk_his_eb_sum_1 = tf.reduce_sum(self.noclk_his_eb, 2)
+            self.noclk_his_eb_sum = tf.reduce_sum(self.noclk_his_eb_sum_1, 1)
+
+    def build_fcn_net(self, inp, use_dice = False):
+        bn1 = tf.layers.batch_normalization(inputs=inp, name='bn1')
+        dnn1 = tf.layers.dense(bn1, 200, activation=None, name='f1')
+        if use_dice:
+            dnn1 = dice(dnn1, name='dice_1')
+        else:
+            dnn1 = prelu(dnn1, 'prelu1')
+
+        dnn2 = tf.layers.dense(dnn1, 80, activation=None, name='f2')
+        if use_dice:
+            dnn2 = dice(dnn2, name='dice_2')
+        else:
+            dnn2 = prelu(dnn2, 'prelu2')
+        dnn3 = tf.layers.dense(dnn2, 2, activation=None, name='f3')
+        self.y_hat = tf.nn.softmax(dnn3) + 0.00000001
+
+        with tf.name_scope('Metrics'):
+            # Cross-entropy loss and optimizer initialization
+            ctr_loss = - tf.reduce_mean(tf.log(self.y_hat) * self.target_ph)
+            self.loss = ctr_loss
+            if self.use_negsampling:
+                self.loss += self.aux_loss
+            tf.summary.scalar('loss', self.loss)
+            self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(self.loss)
+
+            # Accuracy metric
+            self.accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.round(self.y_hat), self.target_ph), tf.float32))
+            tf.summary.scalar('accuracy', self.accuracy)
+
+        self.merged = tf.summary.merge_all()
+
+    def auxiliary_loss(self, h_states, click_seq, noclick_seq, mask, stag = None):
+        mask = tf.cast(mask, tf.float32)
+        click_input_ = tf.concat([h_states, click_seq], -1)
+        noclick_input_ = tf.concat([h_states, noclick_seq], -1)
+        click_prop_ = self.auxiliary_net(click_input_, stag = stag)[:, :, 0]
+        noclick_prop_ = self.auxiliary_net(noclick_input_, stag = stag)[:, :, 0]
+        click_loss_ = - tf.reshape(tf.log(click_prop_), [-1, tf.shape(click_seq)[1]]) * mask
+        noclick_loss_ = - tf.reshape(tf.log(1.0 - noclick_prop_), [-1, tf.shape(noclick_seq)[1]]) * mask
+        loss_ = tf.reduce_mean(click_loss_ + noclick_loss_)
+        return loss_
+
+    def auxiliary_net(self, in_, stag='auxiliary_net'):
+        bn1 = tf.layers.batch_normalization(inputs=in_, name='bn1' + stag, reuse=tf.AUTO_REUSE)
+        dnn1 = tf.layers.dense(bn1, 100, activation=None, name='f1' + stag, reuse=tf.AUTO_REUSE)
+        dnn1 = tf.nn.sigmoid(dnn1)
+        dnn2 = tf.layers.dense(dnn1, 50, activation=None, name='f2' + stag, reuse=tf.AUTO_REUSE)
+        dnn2 = tf.nn.sigmoid(dnn2)
+        dnn3 = tf.layers.dense(dnn2, 2, activation=None, name='f3' + stag, reuse=tf.AUTO_REUSE)
+        y_hat = tf.nn.softmax(dnn3) + 0.00000001
+        return y_hat
+
+
+    def train(self, sess, inps):
+        if self.use_negsampling:
+            loss, accuracy, aux_loss, _ = sess.run([self.loss, self.accuracy, self.aux_loss, self.optimizer], feed_dict={
+                self.uid_batch_ph: inps[0],
+                self.mid_batch_ph: inps[1],
+                self.cat_batch_ph: inps[2],
+                self.mid_his_batch_ph: inps[3],
+                self.cat_his_batch_ph: inps[4],
+                self.mask: inps[5],
+                self.target_ph: inps[6],
+                self.seq_len_ph: inps[7],
+                self.lr: inps[8],
+                self.noclk_mid_batch_ph: inps[9],
+                self.noclk_cat_batch_ph: inps[10],
+            })
+            return loss, accuracy, aux_loss
+        else:
+            loss, accuracy, _ = sess.run([self.loss, self.accuracy, self.optimizer], feed_dict={
+                self.uid_batch_ph: inps[0],
+                self.mid_batch_ph: inps[1],
+                self.cat_batch_ph: inps[2],
+                self.mid_his_batch_ph: inps[3],
+                self.cat_his_batch_ph: inps[4],
+                self.mask: inps[5],
+                self.target_ph: inps[6],
+                self.seq_len_ph: inps[7],
+                self.lr: inps[8],
+            })
+            return loss, accuracy, 0
+
+    def calculate(self, sess, inps):
+        if self.use_negsampling:
+            probs, loss, accuracy, aux_loss = sess.run([self.y_hat, self.loss, self.accuracy, self.aux_loss], feed_dict={
+                self.uid_batch_ph: inps[0],
+                self.mid_batch_ph: inps[1],
+                self.cat_batch_ph: inps[2],
+                self.mid_his_batch_ph: inps[3],
+                self.cat_his_batch_ph: inps[4],
+                self.mask: inps[5],
+                self.target_ph: inps[6],
+                self.seq_len_ph: inps[7],
+                self.noclk_mid_batch_ph: inps[8],
+                self.noclk_cat_batch_ph: inps[9],
+            })
+            return probs, loss, accuracy, aux_loss
+        else:
+            probs, loss, accuracy = sess.run([self.y_hat, self.loss, self.accuracy], feed_dict={
+                self.uid_batch_ph: inps[0],
+                self.mid_batch_ph: inps[1],
+                self.cat_batch_ph: inps[2],
+                self.mid_his_batch_ph: inps[3],
+                self.cat_his_batch_ph: inps[4],
+                self.mask: inps[5],
+                self.target_ph: inps[6],
+                self.seq_len_ph: inps[7]
+            })
+            return probs, loss, accuracy, 0
+
+    def save(self, sess, path):
+        saver = tf.train.Saver()
+        saver.save(sess, save_path=path)
+
+    def restore(self, sess, path):
+        saver = tf.train.Saver()
+        saver.restore(sess, save_path=path)
+        print('model restored from %s' % path)
+
+class Model_DIN_V2_Gru_att_Gru(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=False):
+        super(Model_DIN_V2_Gru_att_Gru, self).__init__(n_uid, n_mid, n_cat,
+                                                       EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE,
+                                                       use_negsampling)
+
+        # RNN layer(-s)
+        with tf.name_scope('rnn_1'):
+            rnn_outputs, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE), inputs=self.item_his_eb,
+                                         sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                         scope="gru1")
+            tf.summary.histogram('GRU_outputs', rnn_outputs)
+
+        # Attention layer
+        with tf.name_scope('Attention_layer_1'):
+            att_outputs, alphas = din_fcn_attention(self.item_eb, rnn_outputs, ATTENTION_SIZE, self.mask,
+                                                    softmax_stag=1, stag='1_1', mode='LIST', return_alphas=True)
+            tf.summary.histogram('alpha_outputs', alphas)
+
+        with tf.name_scope('rnn_2'):
+            rnn_outputs2, final_state2 = dynamic_rnn(GRUCell(HIDDEN_SIZE), inputs=att_outputs,
+                                                     sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                                     scope="gru2")
+            tf.summary.histogram('GRU2_Final_State', final_state2)
+
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum, self.item_eb * self.item_his_eb_sum, final_state2], 1)
+        # Fully connected layer
+        self.build_fcn_net(inp, use_dice=True)
+
+class Model_DIN_V2_Gru_Gru_att(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=False):
+        super(Model_DIN_V2_Gru_Gru_att, self).__init__(n_uid, n_mid, n_cat,
+                                                       EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE,
+                                                       use_negsampling)
+
+        # RNN layer(-s)
+        with tf.name_scope('rnn_1'):
+            rnn_outputs, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE), inputs=self.item_his_eb,
+                                         sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                         scope="gru1")
+            tf.summary.histogram('GRU_outputs', rnn_outputs)
+
+        with tf.name_scope('rnn_2'):
+            rnn_outputs2, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE), inputs=rnn_outputs,
+                                                     sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                                     scope="gru2")
+            tf.summary.histogram('GRU2_outputs', rnn_outputs2)
+
+        # Attention layer
+        with tf.name_scope('Attention_layer_1'):
+            att_outputs, alphas = din_fcn_attention(self.item_eb, rnn_outputs2, ATTENTION_SIZE, self.mask,
+                                                    softmax_stag=1, stag='1_1', mode='LIST', return_alphas=True)
+            att_fea = tf.reduce_sum(att_outputs, 1)
+            tf.summary.histogram('att_fea', att_fea)
+
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum, self.item_eb * self.item_his_eb_sum, att_fea], 1)
+        self.build_fcn_net(inp, use_dice=True)
+
+class Model_WideDeep(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=False):
+        super(Model_WideDeep, self).__init__(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE,
+                                        ATTENTION_SIZE,
+                                        use_negsampling)
+
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum], 1)
+        # Fully connected layer
+        bn1 = tf.layers.batch_normalization(inputs=inp, name='bn1')
+        dnn1 = tf.layers.dense(bn1, 200, activation=None, name='f1')
+        dnn1 = prelu(dnn1, 'p1')
+        dnn2 = tf.layers.dense(dnn1, 80, activation=None, name='f2')
+        dnn2 = prelu(dnn2, 'p2')
+        dnn3 = tf.layers.dense(dnn2, 2, activation=None, name='f3')
+        d_layer_wide = tf.concat([tf.concat([self.item_eb,self.item_his_eb_sum], axis=-1),
+                                self.item_eb * self.item_his_eb_sum], axis=-1)
+        d_layer_wide = tf.layers.dense(d_layer_wide, 2, activation=None, name='f_fm')
+        self.y_hat = tf.nn.softmax(dnn3 + d_layer_wide)
+
+        with tf.name_scope('Metrics'):
+            # Cross-entropy loss and optimizer initialization
+            self.loss = - tf.reduce_mean(tf.log(self.y_hat) * self.target_ph)
+            tf.summary.scalar('loss', self.loss)
+            self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(self.loss)
+
+            # Accuracy metric
+            self.accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.round(self.y_hat), self.target_ph), tf.float32))
+            tf.summary.scalar('accuracy', self.accuracy)
+        self.merged = tf.summary.merge_all()
+
+
+class Model_DIN_V2_Gru_QA_attGru(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=False):
+        super(Model_DIN_V2_Gru_QA_attGru, self).__init__(n_uid, n_mid, n_cat,
+                                                         EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE,
+                                                         use_negsampling)
+
+        # RNN layer(-s)
+        with tf.name_scope('rnn_1'):
+            rnn_outputs, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE), inputs=self.item_his_eb,
+                                         sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                         scope="gru1")
+            tf.summary.histogram('GRU_outputs', rnn_outputs)
+
+        # Attention layer
+        with tf.name_scope('Attention_layer_1'):
+            att_outputs, alphas = din_fcn_attention(self.item_eb, rnn_outputs, ATTENTION_SIZE, self.mask,
+                                                    softmax_stag=1, stag='1_1', mode='LIST', return_alphas=True)
+            tf.summary.histogram('alpha_outputs', alphas)
+
+        with tf.name_scope('rnn_2'):
+            rnn_outputs2, final_state2 = dynamic_rnn(QAAttGRUCell(HIDDEN_SIZE), inputs=rnn_outputs,
+                                                     att_scores = tf.expand_dims(alphas, -1),
+                                                     sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                                     scope="gru2")
+            tf.summary.histogram('GRU2_Final_State', final_state2)
+
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum, self.item_eb * self.item_his_eb_sum, final_state2], 1)
+        self.build_fcn_net(inp, use_dice=True)
+
+class Model_DNN(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=False):
+        super(Model_DNN, self).__init__(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE,
+                                                          ATTENTION_SIZE,
+                                                          use_negsampling)
+
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum], 1)
+        self.build_fcn_net(inp, use_dice=False)
+
+class Model_PNN(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=False):
+        super(Model_PNN, self).__init__(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE,
+                                        ATTENTION_SIZE,
+                                        use_negsampling)
+
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum,
+                         self.item_eb * self.item_his_eb_sum], 1)
+
+        # Fully connected layer
+        self.build_fcn_net(inp, use_dice=False)
+
+
+class Model_DIN(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=False):
+        super(Model_DIN, self).__init__(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE,
+                                           ATTENTION_SIZE,
+                                           use_negsampling)
+
+        # Attention layer
+        with tf.name_scope('Attention_layer'):
+            attention_output = din_attention(self.item_eb, self.item_his_eb, ATTENTION_SIZE, self.mask)
+            att_fea = tf.reduce_sum(attention_output, 1)
+            tf.summary.histogram('att_fea', att_fea)
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum, self.item_eb * self.item_his_eb_sum, att_fea], -1)
+        # Fully connected layer
+        self.build_fcn_net(inp, use_dice=True)
+
+
+class Model_DIN_V2_Gru_Vec_attGru_Neg(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=True):
+        super(Model_DIN_V2_Gru_Vec_attGru_Neg, self).__init__(n_uid, n_mid, n_cat,
+                                                          EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE,
+                                                          use_negsampling)
+
+        # RNN layer(-s)
+        with tf.name_scope('rnn_1'):
+            rnn_outputs, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE), inputs=self.item_his_eb,
+                                         sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                         scope="gru1")
+            tf.summary.histogram('GRU_outputs', rnn_outputs)
+
+        aux_loss_1 = self.auxiliary_loss(rnn_outputs[:, :-1, :], self.item_his_eb[:, 1:, :],
+                                         self.noclk_item_his_eb[:, 1:, :],
+                                         self.mask[:, 1:], stag="gru")
+        self.aux_loss = aux_loss_1
+
+        # Attention layer
+        with tf.name_scope('Attention_layer_1'):
+            att_outputs, alphas = din_fcn_attention(self.item_eb, rnn_outputs, ATTENTION_SIZE, self.mask,
+                                                    softmax_stag=1, stag='1_1', mode='LIST', return_alphas=True)
+            tf.summary.histogram('alpha_outputs', alphas)
+
+        with tf.name_scope('rnn_2'):
+            rnn_outputs2, final_state2 = dynamic_rnn(VecAttGRUCell(HIDDEN_SIZE), inputs=rnn_outputs,
+                                                     att_scores = tf.expand_dims(alphas, -1),
+                                                     sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                                     scope="gru2")
+            tf.summary.histogram('GRU2_Final_State', final_state2)
+
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum, self.item_eb * self.item_his_eb_sum, final_state2], 1)
+        self.build_fcn_net(inp, use_dice=True)
+
+
+class Model_DIN_V2_Gru_Vec_attGru(Model):
+    def __init__(self, n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE, use_negsampling=False):
+        super(Model_DIN_V2_Gru_Vec_attGru, self).__init__(n_uid, n_mid, n_cat,
+                                                          EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE,
+                                                          use_negsampling)
+
+        # RNN layer(-s)
+        with tf.name_scope('rnn_1'):
+            rnn_outputs, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE), inputs=self.item_his_eb,
+                                         sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                         scope="gru1")
+            tf.summary.histogram('GRU_outputs', rnn_outputs)
+
+        # Attention layer
+        with tf.name_scope('Attention_layer_1'):
+            att_outputs, alphas = din_fcn_attention(self.item_eb, rnn_outputs, ATTENTION_SIZE, self.mask,
+                                                    softmax_stag=1, stag='1_1', mode='LIST', return_alphas=True)
+            tf.summary.histogram('alpha_outputs', alphas)
+
+        with tf.name_scope('rnn_2'):
+            rnn_outputs2, final_state2 = dynamic_rnn(VecAttGRUCell(HIDDEN_SIZE), inputs=rnn_outputs,
+                                                     att_scores = tf.expand_dims(alphas, -1),
+                                                     sequence_length=self.seq_len_ph, dtype=tf.float32,
+                                                     scope="gru2")
+            tf.summary.histogram('GRU2_Final_State', final_state2)
+
+        #inp = tf.concat([self.uid_batch_embedded, self.item_eb, final_state2, self.item_his_eb_sum], 1)
+        inp = tf.concat([self.uid_batch_embedded, self.item_eb, self.item_his_eb_sum, self.item_eb * self.item_his_eb_sum, final_state2], 1)
+        self.build_fcn_net(inp, use_dice=True)
diff --git a/recommendation/Basic-DIEN-Demo/source_code/process_data.py b/recommendation/Basic-DIEN-Demo/source_code/process_data.py
new file mode 100755
index 00000000..368a7660
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/process_data.py
@@ -0,0 +1,101 @@
+import sys
+import random
+import time
+
+def process_meta(file):
+    fi = open(file, "r")
+    fo = open("item-info", "w")
+    for line in fi:
+        obj = eval(line)
+        cat = obj["categories"][0][-1]
+        print>>fo, obj["asin"] + "\t" + cat
+
+def process_reviews(file):
+    fi = open(file, "r")
+    user_map = {}
+    fo = open("reviews-info", "w")
+    for line in fi:
+        obj = eval(line)
+        userID = obj["reviewerID"]
+        itemID = obj["asin"]
+        rating = obj["overall"]
+        time = obj["unixReviewTime"]
+        print>>fo, userID + "\t" + itemID + "\t" + str(rating) + "\t" + str(time)
+
+def manual_join():
+    f_rev = open("reviews-info", "r")
+    user_map = {}
+    item_list = []
+    for line in f_rev:
+        line = line.strip()
+        items = line.split("\t")
+        #loctime = time.localtime(float(items[-1]))
+        #items[-1] = time.strftime('%Y-%m-%d', loctime)
+        if items[0] not in user_map:
+            user_map[items[0]]= []
+        user_map[items[0]].append(("\t".join(items), float(items[-1])))
+        item_list.append(items[1])
+    f_meta = open("item-info", "r")
+    meta_map = {}
+    for line in f_meta:
+        arr = line.strip().split("\t")
+        if arr[0] not in meta_map:
+            meta_map[arr[0]] = arr[1]
+            arr = line.strip().split("\t")
+    fo = open("jointed-new", "w")
+    for key in user_map:
+        sorted_user_bh = sorted(user_map[key], key=lambda x:x[1])
+        for line, t in sorted_user_bh:
+            items = line.split("\t")
+            asin = items[1]
+            j = 0
+            while True:
+                asin_neg_index = random.randint(0, len(item_list) - 1)
+                asin_neg = item_list[asin_neg_index]
+                if asin_neg == asin:
+                    continue 
+                items[1] = asin_neg
+                print>>fo, "0" + "\t" + "\t".join(items) + "\t" + meta_map[asin_neg]
+                j += 1
+                if j == 1:             #negative sampling frequency
+                    break
+            if asin in meta_map:
+                print>>fo, "1" + "\t" + line + "\t" + meta_map[asin]
+            else:
+                print>>fo, "1" + "\t" + line + "\t" + "default_cat"
+
+
+def split_test():
+    fi = open("jointed-new", "r")
+    fo = open("jointed-new-split-info", "w")
+    user_count = {}
+    for line in fi:
+        line = line.strip()
+        user = line.split("\t")[1]
+        if user not in user_count:
+            user_count[user] = 0
+        user_count[user] += 1
+    fi.seek(0)
+    i = 0
+    last_user = "A26ZDKC53OP6JD"
+    for line in fi:
+        line = line.strip()
+        user = line.split("\t")[1]
+        if user == last_user:
+            if i < user_count[user] - 2:  # 1 + negative samples
+                print>> fo, "20180118" + "\t" + line
+            else:
+                print>>fo, "20190119" + "\t" + line
+        else:
+            last_user = user
+            i = 0
+            if i < user_count[user] - 2:
+                print>> fo, "20180118" + "\t" + line
+            else:
+                print>>fo, "20190119" + "\t" + line
+        i += 1
+
+process_meta(sys.argv[1])
+process_reviews(sys.argv[2])
+manual_join()
+split_test()
diff --git a/recommendation/Basic-DIEN-Demo/source_code/rnn.py b/recommendation/Basic-DIEN-Demo/source_code/rnn.py
new file mode 100755
index 00000000..ecf7c454
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/rnn.py
@@ -0,0 +1,1453 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""RNN helpers for TensorFlow models.
+
+
+@@bidirectional_dynamic_rnn
+@@dynamic_rnn
+@@raw_rnn
+@@static_rnn
+@@static_state_saving_rnn
+@@static_bidirectional_rnn
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import rnn_cell_impl
+from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.util import nest
+
+
+# pylint: disable=protected-access
+_concat = rnn_cell_impl._concat
+_like_rnncell = rnn_cell_impl._like_rnncell
+# pylint: enable=protected-access
+
+
+def _transpose_batch_time(x):
+  """Transpose the batch and time dimensions of a Tensor.
+
+  Retains as much of the static shape information as possible.
+
+  Args:
+    x: A tensor of rank 2 or higher.
+
+  Returns:
+    x transposed along the first two dimensions.
+
+  Raises:
+    ValueError: if `x` is rank 1 or lower.
+  """
+  x_static_shape = x.get_shape()
+  if x_static_shape.ndims is not None and x_static_shape.ndims < 2:
+    raise ValueError(
+        "Expected input tensor %s to have rank at least 2, but saw shape: %s" %
+        (x, x_static_shape))
+  x_rank = array_ops.rank(x)
+  x_t = array_ops.transpose(
+      x, array_ops.concat(
+          ([1, 0], math_ops.range(2, x_rank)), axis=0))
+  x_t.set_shape(
+      tensor_shape.TensorShape([
+          x_static_shape[1].value, x_static_shape[0].value
+      ]).concatenate(x_static_shape[2:]))
+  return x_t
+
+
+def _best_effort_input_batch_size(flat_input):
+  """Get static input batch size if available, with fallback to the dynamic one.
+
+  Args:
+    flat_input: An iterable of time major input Tensors of shape [max_time,
+      batch_size, ...]. All inputs should have compatible batch sizes.
+
+  Returns:
+    The batch size in Python integer if available, or a scalar Tensor otherwise.
+
+  Raises:
+    ValueError: if there is any input with an invalid shape.
+  """
+  for input_ in flat_input:
+    shape = input_.shape
+    if shape.ndims is None:
+      continue
+    if shape.ndims < 2:
+      raise ValueError(
+          "Expected input tensor %s to have rank at least 2" % input_)
+    batch_size = shape[1].value
+    if batch_size is not None:
+      return batch_size
+  # Fallback to the dynamic batch size of the first input.
+  return array_ops.shape(flat_input[0])[1]
+
+
+def _infer_state_dtype(explicit_dtype, state):
+  """Infer the dtype of an RNN state.
+
+  Args:
+    explicit_dtype: explicitly declared dtype or None.
+    state: RNN's hidden state. Must be a Tensor or a nested iterable containing
+      Tensors.
+
+  Returns:
+    dtype: inferred dtype of hidden state.
+
+  Raises:
+    ValueError: if `state` has heterogeneous dtypes or is empty.
+  """
+  if explicit_dtype is not None:
+    return explicit_dtype
+  elif nest.is_sequence(state):
+    inferred_dtypes = [element.dtype for element in nest.flatten(state)]
+    if not inferred_dtypes:
+      raise ValueError("Unable to infer dtype from empty state.")
+    all_same = all([x == inferred_dtypes[0] for x in inferred_dtypes])
+    if not all_same:
+      raise ValueError(
+          "State has tensors of different inferred_dtypes. Unable to infer a "
+          "single representative dtype.")
+    return inferred_dtypes[0]
+  else:
+    return state.dtype
+
+
+# pylint: disable=unused-argument
+def _rnn_step(
+    time, sequence_length, min_sequence_length, max_sequence_length,
+    zero_output, state, call_cell, state_size, skip_conditionals=False):
+  """Calculate one step of a dynamic RNN minibatch.
+
+  Returns an (output, state) pair conditioned on the sequence_lengths.
+  When skip_conditionals=False, the pseudocode is something like:
+
+  if t >= max_sequence_length:
+    return (zero_output, state)
+  if t < min_sequence_length:
+    return call_cell()
+
+  # Selectively output zeros or output, old state or new state depending
+  # on if we've finished calculating each row.
+  new_output, new_state = call_cell()
+  final_output = np.vstack([
+    zero_output if time >= sequence_lengths[r] else new_output_r
+    for r, new_output_r in enumerate(new_output)
+  ])
+  final_state = np.vstack([
+    state[r] if time >= sequence_lengths[r] else new_state_r
+    for r, new_state_r in enumerate(new_state)
+  ])
+  return (final_output, final_state)
+
+  Args:
+    time: Python int, the current time step
+    sequence_length: int32 `Tensor` vector of size [batch_size]
+    min_sequence_length: int32 `Tensor` scalar, min of sequence_length
+    max_sequence_length: int32 `Tensor` scalar, max of sequence_length
+    zero_output: `Tensor` vector of shape [output_size]
+    state: Either a single `Tensor` matrix of shape `[batch_size, state_size]`,
+      or a list/tuple of such tensors.
+    call_cell: lambda returning tuple of (new_output, new_state) where
+      new_output is a `Tensor` matrix of shape `[batch_size, output_size]`.
+      new_state is a `Tensor` matrix of shape `[batch_size, state_size]`.
+    state_size: The `cell.state_size` associated with the state.
+    skip_conditionals: Python bool, whether to skip using the conditional
+      calculations.  This is useful for `dynamic_rnn`, where the input tensor
+      matches `max_sequence_length`, and using conditionals just slows
+      everything down.
+
+  Returns:
+    A tuple of (`final_output`, `final_state`) as given by the pseudocode above:
+      final_output is a `Tensor` matrix of shape [batch_size, output_size]
+      final_state is either a single `Tensor` matrix, or a tuple of such
+        matrices (matching length and shapes of input `state`).
+
+  Raises:
+    ValueError: If the cell returns a state tuple whose length does not match
+      that returned by `state_size`.
+  """
+
+  # Convert state to a list for ease of use
+  flat_state = nest.flatten(state)
+  flat_zero_output = nest.flatten(zero_output)
+
+  def _copy_one_through(output, new_output):
+    # If the state contains a scalar value we simply pass it through.
+    if output.shape.ndims == 0:
+      return new_output
+    copy_cond = (time >= sequence_length)
+    with ops.colocate_with(new_output):
+      return array_ops.where(copy_cond, output, new_output)
+
+  def _copy_some_through(flat_new_output, flat_new_state):
+    # Use broadcasting select to determine which values should get
+    # the previous state & zero output, and which values should get
+    # a calculated state & output.
+    flat_new_output = [
+        _copy_one_through(zero_output, new_output)
+        for zero_output, new_output in zip(flat_zero_output, flat_new_output)]
+    flat_new_state = [
+        _copy_one_through(state, new_state)
+        for state, new_state in zip(flat_state, flat_new_state)]
+    return flat_new_output + flat_new_state
+
+  def _maybe_copy_some_through():
+    """Run RNN step.  Pass through either no or some past state."""
+    new_output, new_state = call_cell()
+
+    nest.assert_same_structure(state, new_state)
+
+    flat_new_state = nest.flatten(new_state)
+    flat_new_output = nest.flatten(new_output)
+    return control_flow_ops.cond(
+        # if t < min_seq_len: calculate and return everything
+        time < min_sequence_length, lambda: flat_new_output + flat_new_state,
+        # else copy some of it through
+        lambda: _copy_some_through(flat_new_output, flat_new_state))
+
+  # TODO(ebrevdo): skipping these conditionals may cause a slowdown,
+  # but benefits from removing cond() and its gradient.  We should
+  # profile with and without this switch here.
+  if skip_conditionals:
+    # Instead of using conditionals, perform the selective copy at all time
+    # steps.  This is faster when max_seq_len is equal to the number of unrolls
+    # (which is typical for dynamic_rnn).
+    new_output, new_state = call_cell()
+    nest.assert_same_structure(state, new_state)
+    new_state = nest.flatten(new_state)
+    new_output = nest.flatten(new_output)
+    final_output_and_state = _copy_some_through(new_output, new_state)
+  else:
+    empty_update = lambda: flat_zero_output + flat_state
+    final_output_and_state = control_flow_ops.cond(
+        # if t >= max_seq_len: copy all state through, output zeros
+        time >= max_sequence_length, empty_update,
+        # otherwise calculation is required: copy some or all of it through
+        _maybe_copy_some_through)
+
+  if len(final_output_and_state) != len(flat_zero_output) + len(flat_state):
+    raise ValueError("Internal error: state and output were not concatenated "
+                     "correctly.")
+  final_output = final_output_and_state[:len(flat_zero_output)]
+  final_state = final_output_and_state[len(flat_zero_output):]
+
+  for output, flat_output in zip(final_output, flat_zero_output):
+    output.set_shape(flat_output.get_shape())
+  for substate, flat_substate in zip(final_state, flat_state):
+    substate.set_shape(flat_substate.get_shape())
+
+  final_output = nest.pack_sequence_as(
+      structure=zero_output, flat_sequence=final_output)
+  final_state = nest.pack_sequence_as(
+      structure=state, flat_sequence=final_state)
+
+  return final_output, final_state
+
+
+def _reverse_seq(input_seq, lengths):
+  """Reverse a list of Tensors up to specified lengths.
+
+  Args:
+    input_seq: Sequence of seq_len tensors of dimension (batch_size, n_features)
+               or nested tuples of tensors.
+    lengths:   A `Tensor` of dimension batch_size, containing lengths for each
+               sequence in the batch. If "None" is specified, simply reverses
+               the list.
+
+  Returns:
+    time-reversed sequence
+  """
+  if lengths is None:
+    return list(reversed(input_seq))
+
+  flat_input_seq = tuple(nest.flatten(input_) for input_ in input_seq)
+
+  flat_results = [[] for _ in range(len(input_seq))]
+  for sequence in zip(*flat_input_seq):
+    input_shape = tensor_shape.unknown_shape(
+        ndims=sequence[0].get_shape().ndims)
+    for input_ in sequence:
+      input_shape.merge_with(input_.get_shape())
+      input_.set_shape(input_shape)
+
+    # Join into (time, batch_size, depth)
+    s_joined = array_ops.stack(sequence)
+
+    # Reverse along dimension 0
+    s_reversed = array_ops.reverse_sequence(s_joined, lengths, 0, 1)
+    # Split again into list
+    result = array_ops.unstack(s_reversed)
+    for r, flat_result in zip(result, flat_results):
+      r.set_shape(input_shape)
+      flat_result.append(r)
+
+  results = [nest.pack_sequence_as(structure=input_, flat_sequence=flat_result)
+             for input_, flat_result in zip(input_seq, flat_results)]
+  return results
+
+
+def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None,
+                              initial_state_fw=None, initial_state_bw=None,
+                              dtype=None, parallel_iterations=None,
+                              swap_memory=False, time_major=False, scope=None):
+  """Creates a dynamic version of bidirectional recurrent neural network.
+
+  Takes input and builds independent forward and backward RNNs. The input_size
+  of forward and backward cell must match. The initial state for both directions
+  is zero by default (but can be set optionally) and no intermediate states are
+  ever returned -- the network is fully unrolled for the given (passed in)
+  length(s) of the sequence(s) or completely unrolled if length(s) is not
+  given.
+
+  Args:
+    cell_fw: An instance of RNNCell, to be used for forward direction.
+    cell_bw: An instance of RNNCell, to be used for backward direction.
+    inputs: The RNN inputs.
+      If time_major == False (default), this must be a tensor of shape:
+        `[batch_size, max_time, ...]`, or a nested tuple of such elements.
+      If time_major == True, this must be a tensor of shape:
+        `[max_time, batch_size, ...]`, or a nested tuple of such elements.
+    sequence_length: (optional) An int32/int64 vector, size `[batch_size]`,
+      containing the actual lengths for each of the sequences in the batch.
+      If not provided, all batch entries are assumed to be full sequences; and
+      time reversal is applied from time `0` to `max_time` for each sequence.
+    initial_state_fw: (optional) An initial state for the forward RNN.
+      This must be a tensor of appropriate type and shape
+      `[batch_size, cell_fw.state_size]`.
+      If `cell_fw.state_size` is a tuple, this should be a tuple of
+      tensors having shapes `[batch_size, s] for s in cell_fw.state_size`.
+    initial_state_bw: (optional) Same as for `initial_state_fw`, but using
+      the corresponding properties of `cell_bw`.
+    dtype: (optional) The data type for the initial states and expected output.
+      Required if initial_states are not provided or RNN states have a
+      heterogeneous dtype.
+    parallel_iterations: (Default: 32).  The number of iterations to run in
+      parallel.  Those operations which do not have any temporal dependency
+      and can be run in parallel, will be.  This parameter trades off
+      time for space.  Values >> 1 use more memory but take less time,
+      while smaller values use less memory but computations take longer.
+    swap_memory: Transparently swap the tensors produced in forward inference
+      but needed for back prop from GPU to CPU.  This allows training RNNs
+      which would typically not fit on a single GPU, with very minimal (or no)
+      performance penalty.
+    time_major: The shape format of the `inputs` and `outputs` Tensors.
+      If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
+      If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
+      Using `time_major = True` is a bit more efficient because it avoids
+      transposes at the beginning and end of the RNN calculation.  However,
+      most TensorFlow data is batch-major, so by default this function
+      accepts input and emits output in batch-major form.
+    scope: VariableScope for the created subgraph; defaults to
+      "bidirectional_rnn"
+
+  Returns:
+    A tuple (outputs, output_states) where:
+      outputs: A tuple (output_fw, output_bw) containing the forward and
+        the backward rnn output `Tensor`.
+        If time_major == False (default),
+          output_fw will be a `Tensor` shaped:
+          `[batch_size, max_time, cell_fw.output_size]`
+          and output_bw will be a `Tensor` shaped:
+          `[batch_size, max_time, cell_bw.output_size]`.
+        If time_major == True,
+          output_fw will be a `Tensor` shaped:
+          `[max_time, batch_size, cell_fw.output_size]`
+          and output_bw will be a `Tensor` shaped:
+          `[max_time, batch_size, cell_bw.output_size]`.
+        It returns a tuple instead of a single concatenated `Tensor`, unlike
+        in the `bidirectional_rnn`. If the concatenated one is preferred,
+        the forward and backward outputs can be concatenated as
+        `tf.concat(outputs, 2)`.
+      output_states: A tuple (output_state_fw, output_state_bw) containing
+        the forward and the backward final states of bidirectional rnn.
+
+  Raises:
+    TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`.
+  """
+
+  if not _like_rnncell(cell_fw):
+    raise TypeError("cell_fw must be an instance of RNNCell")
+  if not _like_rnncell(cell_bw):
+    raise TypeError("cell_bw must be an instance of RNNCell")
+
+  with vs.variable_scope(scope or "bidirectional_rnn"):
+    # Forward direction
+    with vs.variable_scope("fw") as fw_scope:
+      output_fw, output_state_fw = dynamic_rnn(
+          cell=cell_fw, inputs=inputs, sequence_length=sequence_length,
+          initial_state=initial_state_fw, dtype=dtype,
+          parallel_iterations=parallel_iterations, swap_memory=swap_memory,
+          time_major=time_major, scope=fw_scope)
+
+    # Backward direction
+    if not time_major:
+      time_dim = 1
+      batch_dim = 0
+    else:
+      time_dim = 0
+      batch_dim = 1
+
+    def _reverse(input_, seq_lengths, seq_dim, batch_dim):
+      if seq_lengths is not None:
+        return array_ops.reverse_sequence(
+            input=input_, seq_lengths=seq_lengths,
+            seq_dim=seq_dim, batch_dim=batch_dim)
+      else:
+        return array_ops.reverse(input_, axis=[seq_dim])
+
+    with vs.variable_scope("bw") as bw_scope:
+      inputs_reverse = _reverse(
+          inputs, seq_lengths=sequence_length,
+          seq_dim=time_dim, batch_dim=batch_dim)
+      tmp, output_state_bw = dynamic_rnn(
+          cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length,
+          initial_state=initial_state_bw, dtype=dtype,
+          parallel_iterations=parallel_iterations, swap_memory=swap_memory,
+          time_major=time_major, scope=bw_scope)
+
+  output_bw = _reverse(
+      tmp, seq_lengths=sequence_length,
+      seq_dim=time_dim, batch_dim=batch_dim)
+
+  outputs = (output_fw, output_bw)
+  output_states = (output_state_fw, output_state_bw)
+
+  return (outputs, output_states)
+
+
+def dynamic_rnn(cell, inputs, att_scores=None, sequence_length=None, initial_state=None,
+                dtype=None, parallel_iterations=None, swap_memory=False,
+                time_major=False, scope=None):
+  """Creates a recurrent neural network specified by RNNCell `cell`.
+
+  Performs fully dynamic unrolling of `inputs`.
+
+  Example:
+
+  ```python
+  # create a BasicRNNCell
+  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
+
+  # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size]
+
+  # defining initial state
+  initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32)
+
+  # 'state' is a tensor of shape [batch_size, cell_state_size]
+  outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data,
+                                     initial_state=initial_state,
+                                     dtype=tf.float32)
+  ```
+
+  ```python
+  # create 2 LSTMCells
+  rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]]
+
+  # create a RNN cell composed sequentially of a number of RNNCells
+  multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)
+
+  # 'outputs' is a tensor of shape [batch_size, max_time, 256]
+  # 'state' is a N-tuple where N is the number of LSTMCells containing a
+  # tf.contrib.rnn.LSTMStateTuple for each cell
+  outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell,
+                                     inputs=data,
+                                     dtype=tf.float32)
+  ```
+
+
+  Args:
+    cell: An instance of RNNCell.
+    inputs: The RNN inputs.
+      If `time_major == False` (default), this must be a `Tensor` of shape:
+        `[batch_size, max_time, ...]`, or a nested tuple of such
+        elements.
+      If `time_major == True`, this must be a `Tensor` of shape:
+        `[max_time, batch_size, ...]`, or a nested tuple of such
+        elements.
+      This may also be a (possibly nested) tuple of Tensors satisfying
+      this property.  The first two dimensions must match across all the inputs,
+      but otherwise the ranks and other shape components may differ.
+      In this case, input to `cell` at each time-step will replicate the
+      structure of these tuples, except for the time dimension (from which the
+      time is taken).
+      The input to `cell` at each time step will be a `Tensor` or (possibly
+      nested) tuple of Tensors each with dimensions `[batch_size, ...]`.
+    sequence_length: (optional) An int32/int64 vector sized `[batch_size]`.
+      Used to copy-through state and zero-out outputs when past a batch
+      element's sequence length.  So it's more for correctness than performance.
+    initial_state: (optional) An initial state for the RNN.
+      If `cell.state_size` is an integer, this must be
+      a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`.
+      If `cell.state_size` is a tuple, this should be a tuple of
+      tensors having shapes `[batch_size, s] for s in cell.state_size`.
+    dtype: (optional) The data type for the initial state and expected output.
+      Required if initial_state is not provided or RNN state has a heterogeneous
+      dtype.
+    parallel_iterations: (Default: 32).  The number of iterations to run in
+      parallel.  Those operations which do not have any temporal dependency
+      and can be run in parallel, will be.  This parameter trades off
+      time for space.  Values >> 1 use more memory but take less time,
+      while smaller values use less memory but computations take longer.
+    swap_memory: Transparently swap the tensors produced in forward inference
+      but needed for back prop from GPU to CPU.  This allows training RNNs
+      which would typically not fit on a single GPU, with very minimal (or no)
+      performance penalty.
+    time_major: The shape format of the `inputs` and `outputs` Tensors.
+      If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
+      If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
+      Using `time_major = True` is a bit more efficient because it avoids
+      transposes at the beginning and end of the RNN calculation.  However,
+      most TensorFlow data is batch-major, so by default this function
+      accepts input and emits output in batch-major form.
+    scope: VariableScope for the created subgraph; defaults to "rnn".
+
+  Returns:
+    A pair (outputs, state) where:
+
+    outputs: The RNN output `Tensor`.
+
+      If time_major == False (default), this will be a `Tensor` shaped:
+        `[batch_size, max_time, cell.output_size]`.
+
+      If time_major == True, this will be a `Tensor` shaped:
+        `[max_time, batch_size, cell.output_size]`.
+
+      Note, if `cell.output_size` is a (possibly nested) tuple of integers
+      or `TensorShape` objects, then `outputs` will be a tuple having the
+      same structure as `cell.output_size`, containing Tensors having shapes
+      corresponding to the shape data in `cell.output_size`.
+
+    state: The final state.  If `cell.state_size` is an int, this
+      will be shaped `[batch_size, cell.state_size]`.  If it is a
+      `TensorShape`, this will be shaped `[batch_size] + cell.state_size`.
+      If it is a (possibly nested) tuple of ints or `TensorShape`, this will
+      be a tuple having the corresponding shapes. If cells are `LSTMCells`
+      `state` will be a tuple containing a `LSTMStateTuple` for each cell.
+
+  Raises:
+    TypeError: If `cell` is not an instance of RNNCell.
+    ValueError: If inputs is None or an empty list.
+  """
+  if not _like_rnncell(cell):
+    raise TypeError("cell must be an instance of RNNCell")
+
+  # By default, time_major==False and inputs are batch-major: shaped
+  #   [batch, time, depth]
+  # For internal calculations, we transpose to [time, batch, depth]
+  flat_input = nest.flatten(inputs)
+
+  if not time_major:
+    # (B,T,D) => (T,B,D)
+    flat_input = [ops.convert_to_tensor(input_) for input_ in flat_input]
+    flat_input = tuple(_transpose_batch_time(input_) for input_ in flat_input)
+
+  parallel_iterations = parallel_iterations or 32
+  if sequence_length is not None:
+    sequence_length = math_ops.to_int32(sequence_length)
+    if sequence_length.get_shape().ndims not in (None, 1):
+      raise ValueError(
+          "sequence_length must be a vector of length batch_size, "
+          "but saw shape: %s" % sequence_length.get_shape())
+    sequence_length = array_ops.identity(  # Just to find it in the graph.
+        sequence_length, name="sequence_length")
+
+  # Create a new scope in which the caching device is either
+  # determined by the parent scope, or is set to place the cached
+  # Variable using the same placement as for the rest of the RNN.
+  with vs.variable_scope(scope or "rnn") as varscope:
+    if varscope.caching_device is None:
+      varscope.set_caching_device(lambda op: op.device)
+    batch_size = _best_effort_input_batch_size(flat_input)
+
+    if initial_state is not None:
+      state = initial_state
+    else:
+      if not dtype:
+        raise ValueError("If there is no initial_state, you must give a dtype.")
+      state = cell.zero_state(batch_size, dtype)
+
+    def _assert_has_shape(x, shape):
+      x_shape = array_ops.shape(x)
+      packed_shape = array_ops.stack(shape)
+      return control_flow_ops.Assert(
+          math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)),
+          ["Expected shape for Tensor %s is " % x.name,
+           packed_shape, " but saw shape: ", x_shape])
+
+    if sequence_length is not None:
+      # Perform some shape validation
+      with ops.control_dependencies(
+          [_assert_has_shape(sequence_length, [batch_size])]):
+        sequence_length = array_ops.identity(
+            sequence_length, name="CheckSeqLen")
+
+    inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input)
+
+    (outputs, final_state) = _dynamic_rnn_loop(
+        cell,
+        inputs,
+        state,
+        parallel_iterations=parallel_iterations,
+        swap_memory=swap_memory,
+        att_scores = att_scores,
+        sequence_length=sequence_length,
+        dtype=dtype)
+
+    # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].
+    # If we are performing batch-major calculations, transpose output back
+    # to shape [batch, time, depth]
+    if not time_major:
+      # (T,B,D) => (B,T,D)
+      outputs = nest.map_structure(_transpose_batch_time, outputs)
+
+    return (outputs, final_state)
+
+
+def _dynamic_rnn_loop(cell,
+                      inputs,
+                      initial_state,
+                      parallel_iterations,
+                      swap_memory,
+                      att_scores = None,
+                      sequence_length=None,
+                      dtype=None):
+  """Internal implementation of Dynamic RNN.
+
+  Args:
+    cell: An instance of RNNCell.
+    inputs: A `Tensor` of shape [time, batch_size, input_size], or a nested
+      tuple of such elements.
+    initial_state: A `Tensor` of shape `[batch_size, state_size]`, or if
+      `cell.state_size` is a tuple, then this should be a tuple of
+      tensors having shapes `[batch_size, s] for s in cell.state_size`.
+    parallel_iterations: Positive Python int.
+    swap_memory: A Python boolean
+    sequence_length: (optional) An `int32` `Tensor` of shape [batch_size].
+    dtype: (optional) Expected dtype of output. If not specified, inferred from
+      initial_state.
+
+  Returns:
+    Tuple `(final_outputs, final_state)`.
+    final_outputs:
+      A `Tensor` of shape `[time, batch_size, cell.output_size]`.  If
+      `cell.output_size` is a (possibly nested) tuple of ints or `TensorShape`
+      objects, then this returns a (possibly nsted) tuple of Tensors matching
+      the corresponding shapes.
+    final_state:
+      A `Tensor`, or possibly nested tuple of Tensors, matching in length
+      and shapes to `initial_state`.
+
+  Raises:
+    ValueError: If the input depth cannot be inferred via shape inference
+      from the inputs.
+  """
+  state = initial_state
+  assert isinstance(parallel_iterations, int), "parallel_iterations must be int"
+
+  state_size = cell.state_size
+
+  flat_input = nest.flatten(inputs)
+  flat_output_size = nest.flatten(cell.output_size)
+
+  # Construct an initial output
+  input_shape = array_ops.shape(flat_input[0])
+  time_steps = input_shape[0]
+  batch_size = _best_effort_input_batch_size(flat_input)
+
+  inputs_got_shape = tuple(input_.get_shape().with_rank_at_least(3)
+                           for input_ in flat_input)
+
+  const_time_steps, const_batch_size = inputs_got_shape[0].as_list()[:2]
+
+  for shape in inputs_got_shape:
+    if not shape[2:].is_fully_defined():
+      raise ValueError(
+          "Input size (depth of inputs) must be accessible via shape inference,"
+          " but saw value None.")
+    got_time_steps = shape[0].value
+    got_batch_size = shape[1].value
+    if const_time_steps != got_time_steps:
+      raise ValueError(
+          "Time steps is not the same for all the elements in the input in a "
+          "batch.")
+    if const_batch_size != got_batch_size:
+      raise ValueError(
+          "Batch_size is not the same for all the elements in the input.")
+
+  # Prepare dynamic conditional copying of state & output
+  def _create_zero_arrays(size):
+    size = _concat(batch_size, size)
+    return array_ops.zeros(
+        array_ops.stack(size), _infer_state_dtype(dtype, state))
+
+  flat_zero_output = tuple(_create_zero_arrays(output)
+                           for output in flat_output_size)
+  zero_output = nest.pack_sequence_as(structure=cell.output_size,
+                                      flat_sequence=flat_zero_output)
+
+  if sequence_length is not None:
+    min_sequence_length = math_ops.reduce_min(sequence_length)
+    max_sequence_length = math_ops.reduce_max(sequence_length)
+
+  time = array_ops.constant(0, dtype=dtypes.int32, name="time")
+
+  with ops.name_scope("dynamic_rnn") as scope:
+    base_name = scope
+
+  def _create_ta(name, dtype):
+    return tensor_array_ops.TensorArray(dtype=dtype,
+                                        size=time_steps,
+                                        tensor_array_name=base_name + name)
+
+  output_ta = tuple(_create_ta("output_%d" % i,
+                               _infer_state_dtype(dtype, state))
+                    for i in range(len(flat_output_size)))
+  input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype)
+                   for i in range(len(flat_input)))
+
+  input_ta = tuple(ta.unstack(input_)
+                   for ta, input_ in zip(input_ta, flat_input))
+
+  def _time_step(time, output_ta_t, state, att_scores=None):
+    """Take a time step of the dynamic RNN.
+
+    Args:
+      time: int32 scalar Tensor.
+      output_ta_t: List of `TensorArray`s that represent the output.
+      state: nested tuple of vector tensors that represent the state.
+
+    Returns:
+      The tuple (time + 1, output_ta_t with updated flow, new_state).
+    """
+
+    input_t = tuple(ta.read(time) for ta in input_ta)
+    # Restore some shape information
+    for input_, shape in zip(input_t, inputs_got_shape):
+      input_.set_shape(shape[1:])
+
+    input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t)
+    if att_scores is not None:
+        att_score = att_scores[:, time, :]
+        call_cell = lambda: cell(input_t, state, att_score)
+    else:
+        call_cell = lambda: cell(input_t, state)
+
+    if sequence_length is not None:
+      (output, new_state) = _rnn_step(
+          time=time,
+          sequence_length=sequence_length,
+          min_sequence_length=min_sequence_length,
+          max_sequence_length=max_sequence_length,
+          zero_output=zero_output,
+          state=state,
+          call_cell=call_cell,
+          state_size=state_size,
+          skip_conditionals=True)
+    else:
+      (output, new_state) = call_cell()
+
+    # Pack state if using state tuples
+    output = nest.flatten(output)
+
+    output_ta_t = tuple(
+        ta.write(time, out) for ta, out in zip(output_ta_t, output))
+    if att_scores is not None:
+        return (time + 1, output_ta_t, new_state, att_scores)
+    else:
+        return (time + 1, output_ta_t, new_state)
+
+  if att_scores is not None:  
+      _, output_final_ta, final_state, _ = control_flow_ops.while_loop(
+          cond=lambda time, *_: time < time_steps,
+          body=_time_step,
+          loop_vars=(time, output_ta, state, att_scores),
+          parallel_iterations=parallel_iterations,
+          swap_memory=swap_memory)
+  else:
+      _, output_final_ta, final_state = control_flow_ops.while_loop(
+          cond=lambda time, *_: time < time_steps,
+          body=_time_step,
+          loop_vars=(time, output_ta, state),
+          parallel_iterations=parallel_iterations,
+          swap_memory=swap_memory)
+
+  # Unpack final output if not using output tuples.
+  final_outputs = tuple(ta.stack() for ta in output_final_ta)
+
+  # Restore some shape information
+  for output, output_size in zip(final_outputs, flat_output_size):
+    shape = _concat(
+        [const_time_steps, const_batch_size], output_size, static=True)
+    output.set_shape(shape)
+
+  final_outputs = nest.pack_sequence_as(
+      structure=cell.output_size, flat_sequence=final_outputs)
+
+  return (final_outputs, final_state)
+
+
+def raw_rnn(cell, loop_fn,
+            parallel_iterations=None, swap_memory=False, scope=None):
+  """Creates an `RNN` specified by RNNCell `cell` and loop function `loop_fn`.
+
+  **NOTE: This method is still in testing, and the API may change.**
+
+  This function is a more primitive version of `dynamic_rnn` that provides
+  more direct access to the inputs each iteration.  It also provides more
+  control over when to start and finish reading the sequence, and
+  what to emit for the output.
+
+  For example, it can be used to implement the dynamic decoder of a seq2seq
+  model.
+
+  Instead of working with `Tensor` objects, most operations work with
+  `TensorArray` objects directly.
+
+  The operation of `raw_rnn`, in pseudo-code, is basically the following:
+
+  ```python
+  time = tf.constant(0, dtype=tf.int32)
+  (finished, next_input, initial_state, _, loop_state) = loop_fn(
+      time=time, cell_output=None, cell_state=None, loop_state=None)
+  emit_ta = TensorArray(dynamic_size=True, dtype=initial_state.dtype)
+  state = initial_state
+  while not all(finished):
+    (output, cell_state) = cell(next_input, state)
+    (next_finished, next_input, next_state, emit, loop_state) = loop_fn(
+        time=time + 1, cell_output=output, cell_state=cell_state,
+        loop_state=loop_state)
+    # Emit zeros and copy forward state for minibatch entries that are finished.
+    state = tf.where(finished, state, next_state)
+    emit = tf.where(finished, tf.zeros_like(emit), emit)
+    emit_ta = emit_ta.write(time, emit)
+    # If any new minibatch entries are marked as finished, mark these.
+    finished = tf.logical_or(finished, next_finished)
+    time += 1
+  return (emit_ta, state, loop_state)
+  ```
+
+  with the additional properties that output and state may be (possibly nested)
+  tuples, as determined by `cell.output_size` and `cell.state_size`, and
+  as a result the final `state` and `emit_ta` may themselves be tuples.
+
+  A simple implementation of `dynamic_rnn` via `raw_rnn` looks like this:
+
+  ```python
+  inputs = tf.placeholder(shape=(max_time, batch_size, input_depth),
+                          dtype=tf.float32)
+  sequence_length = tf.placeholder(shape=(batch_size,), dtype=tf.int32)
+  inputs_ta = tf.TensorArray(dtype=tf.float32, size=max_time)
+  inputs_ta = inputs_ta.unstack(inputs)
+
+  cell = tf.contrib.rnn.LSTMCell(num_units)
+
+  def loop_fn(time, cell_output, cell_state, loop_state):
+    emit_output = cell_output  # == None for time == 0
+    if cell_output is None:  # time == 0
+      next_cell_state = cell.zero_state(batch_size, tf.float32)
+    else:
+      next_cell_state = cell_state
+    elements_finished = (time >= sequence_length)
+    finished = tf.reduce_all(elements_finished)
+    next_input = tf.cond(
+        finished,
+        lambda: tf.zeros([batch_size, input_depth], dtype=tf.float32),
+        lambda: inputs_ta.read(time))
+    next_loop_state = None
+    return (elements_finished, next_input, next_cell_state,
+            emit_output, next_loop_state)
+
+  outputs_ta, final_state, _ = raw_rnn(cell, loop_fn)
+  outputs = outputs_ta.stack()
+  ```
+
+  Args:
+    cell: An instance of RNNCell.
+    loop_fn: A callable that takes inputs
+      `(time, cell_output, cell_state, loop_state)`
+      and returns the tuple
+      `(finished, next_input, next_cell_state, emit_output, next_loop_state)`.
+      Here `time` is an int32 scalar `Tensor`, `cell_output` is a
+      `Tensor` or (possibly nested) tuple of tensors as determined by
+      `cell.output_size`, and `cell_state` is a `Tensor`
+      or (possibly nested) tuple of tensors, as determined by the `loop_fn`
+      on its first call (and should match `cell.state_size`).
+      The outputs are: `finished`, a boolean `Tensor` of
+      shape `[batch_size]`, `next_input`: the next input to feed to `cell`,
+      `next_cell_state`: the next state to feed to `cell`,
+      and `emit_output`: the output to store for this iteration.
+
+      Note that `emit_output` should be a `Tensor` or (possibly nested)
+      tuple of tensors with shapes and structure matching `cell.output_size`
+      and `cell_output` above.  The parameter `cell_state` and output
+      `next_cell_state` may be either a single or (possibly nested) tuple
+      of tensors.  The parameter `loop_state` and
+      output `next_loop_state` may be either a single or (possibly nested) tuple
+      of `Tensor` and `TensorArray` objects.  This last parameter
+      may be ignored by `loop_fn` and the return value may be `None`.  If it
+      is not `None`, then the `loop_state` will be propagated through the RNN
+      loop, for use purely by `loop_fn` to keep track of its own state.
+      The `next_loop_state` parameter returned may be `None`.
+
+      The first call to `loop_fn` will be `time = 0`, `cell_output = None`,
+      `cell_state = None`, and `loop_state = None`.  For this call:
+      The `next_cell_state` value should be the value with which to initialize
+      the cell's state.  It may be a final state from a previous RNN or it
+      may be the output of `cell.zero_state()`.  It should be a
+      (possibly nested) tuple structure of tensors.
+      If `cell.state_size` is an integer, this must be
+      a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`.
+      If `cell.state_size` is a `TensorShape`, this must be a `Tensor` of
+      appropriate type and shape `[batch_size] + cell.state_size`.
+      If `cell.state_size` is a (possibly nested) tuple of ints or
+      `TensorShape`, this will be a tuple having the corresponding shapes.
+      The `emit_output` value may be either `None` or a (possibly nested)
+      tuple structure of tensors, e.g.,
+      `(tf.zeros(shape_0, dtype=dtype_0), tf.zeros(shape_1, dtype=dtype_1))`.
+      If this first `emit_output` return value is `None`,
+      then the `emit_ta` result of `raw_rnn` will have the same structure and
+      dtypes as `cell.output_size`.  Otherwise `emit_ta` will have the same
+      structure, shapes (prepended with a `batch_size` dimension), and dtypes
+      as `emit_output`.  The actual values returned for `emit_output` at this
+      initializing call are ignored.  Note, this emit structure must be
+      consistent across all time steps.
+
+    parallel_iterations: (Default: 32).  The number of iterations to run in
+      parallel.  Those operations which do not have any temporal dependency
+      and can be run in parallel, will be.  This parameter trades off
+      time for space.  Values >> 1 use more memory but take less time,
+      while smaller values use less memory but computations take longer.
+    swap_memory: Transparently swap the tensors produced in forward inference
+      but needed for back prop from GPU to CPU.  This allows training RNNs
+      which would typically not fit on a single GPU, with very minimal (or no)
+      performance penalty.
+    scope: VariableScope for the created subgraph; defaults to "rnn".
+
+  Returns:
+    A tuple `(emit_ta, final_state, final_loop_state)` where:
+
+    `emit_ta`: The RNN output `TensorArray`.
+       If `loop_fn` returns a (possibly nested) set of Tensors for
+       `emit_output` during initialization, (inputs `time = 0`,
+       `cell_output = None`, and `loop_state = None`), then `emit_ta` will
+       have the same structure, dtypes, and shapes as `emit_output` instead.
+       If `loop_fn` returns `emit_output = None` during this call,
+       the structure of `cell.output_size` is used:
+       If `cell.output_size` is a (possibly nested) tuple of integers
+       or `TensorShape` objects, then `emit_ta` will be a tuple having the
+       same structure as `cell.output_size`, containing TensorArrays whose
+       elements' shapes correspond to the shape data in `cell.output_size`.
+
+    `final_state`: The final cell state.  If `cell.state_size` is an int, this
+      will be shaped `[batch_size, cell.state_size]`.  If it is a
+      `TensorShape`, this will be shaped `[batch_size] + cell.state_size`.
+      If it is a (possibly nested) tuple of ints or `TensorShape`, this will
+      be a tuple having the corresponding shapes.
+
+    `final_loop_state`: The final loop state as returned by `loop_fn`.
+
+  Raises:
+    TypeError: If `cell` is not an instance of RNNCell, or `loop_fn` is not
+      a `callable`.
+  """
+
+  if not _like_rnncell(cell):
+    raise TypeError("cell must be an instance of RNNCell")
+  if not callable(loop_fn):
+    raise TypeError("loop_fn must be a callable")
+
+  parallel_iterations = parallel_iterations or 32
+
+  # Create a new scope in which the caching device is either
+  # determined by the parent scope, or is set to place the cached
+  # Variable using the same placement as for the rest of the RNN.
+  with vs.variable_scope(scope or "rnn") as varscope:
+    if varscope.caching_device is None:
+      varscope.set_caching_device(lambda op: op.device)
+
+    time = constant_op.constant(0, dtype=dtypes.int32)
+    (elements_finished, next_input, initial_state, emit_structure,
+     init_loop_state) = loop_fn(
+         time, None, None, None)  # time, cell_output, cell_state, loop_state
+    flat_input = nest.flatten(next_input)
+
+    # Need a surrogate loop state for the while_loop if none is available.
+    loop_state = (init_loop_state if init_loop_state is not None
+                  else constant_op.constant(0, dtype=dtypes.int32))
+
+    input_shape = [input_.get_shape() for input_ in flat_input]
+    static_batch_size = input_shape[0][0]
+
+    for input_shape_i in input_shape:
+      # Static verification that batch sizes all match
+      static_batch_size.merge_with(input_shape_i[0])
+
+    batch_size = static_batch_size.value
+    if batch_size is None:
+      batch_size = array_ops.shape(flat_input[0])[0]
+
+    nest.assert_same_structure(initial_state, cell.state_size)
+    state = initial_state
+    flat_state = nest.flatten(state)
+    flat_state = [ops.convert_to_tensor(s) for s in flat_state]
+    state = nest.pack_sequence_as(structure=state,
+                                  flat_sequence=flat_state)
+
+    if emit_structure is not None:
+      flat_emit_structure = nest.flatten(emit_structure)
+      flat_emit_size = [emit.shape if emit.shape.is_fully_defined() else
+                        array_ops.shape(emit) for emit in flat_emit_structure]
+      flat_emit_dtypes = [emit.dtype for emit in flat_emit_structure]
+    else:
+      emit_structure = cell.output_size
+      flat_emit_size = nest.flatten(emit_structure)
+      flat_emit_dtypes = [flat_state[0].dtype] * len(flat_emit_size)
+
+    flat_emit_ta = [
+        tensor_array_ops.TensorArray(
+            dtype=dtype_i, dynamic_size=True, size=0, name="rnn_output_%d" % i)
+        for i, dtype_i in enumerate(flat_emit_dtypes)]
+    emit_ta = nest.pack_sequence_as(structure=emit_structure,
+                                    flat_sequence=flat_emit_ta)
+    flat_zero_emit = [
+        array_ops.zeros(_concat(batch_size, size_i), dtype_i)
+        for size_i, dtype_i in zip(flat_emit_size, flat_emit_dtypes)]
+    zero_emit = nest.pack_sequence_as(structure=emit_structure,
+                                      flat_sequence=flat_zero_emit)
+
+    def condition(unused_time, elements_finished, *_):
+      return math_ops.logical_not(math_ops.reduce_all(elements_finished))
+
+    def body(time, elements_finished, current_input,
+             emit_ta, state, loop_state):
+      """Internal while loop body for raw_rnn.
+
+      Args:
+        time: time scalar.
+        elements_finished: batch-size vector.
+        current_input: possibly nested tuple of input tensors.
+        emit_ta: possibly nested tuple of output TensorArrays.
+        state: possibly nested tuple of state tensors.
+        loop_state: possibly nested tuple of loop state tensors.
+
+      Returns:
+        Tuple having the same size as Args but with updated values.
+      """
+      (next_output, cell_state) = cell(current_input, state)
+
+      nest.assert_same_structure(state, cell_state)
+      nest.assert_same_structure(cell.output_size, next_output)
+
+      next_time = time + 1
+      (next_finished, next_input, next_state, emit_output,
+       next_loop_state) = loop_fn(
+           next_time, next_output, cell_state, loop_state)
+
+      nest.assert_same_structure(state, next_state)
+      nest.assert_same_structure(current_input, next_input)
+      nest.assert_same_structure(emit_ta, emit_output)
+
+      # If loop_fn returns None for next_loop_state, just reuse the
+      # previous one.
+      loop_state = loop_state if next_loop_state is None else next_loop_state
+
+      def _copy_some_through(current, candidate):
+        """Copy some tensors through via array_ops.where."""
+        def copy_fn(cur_i, cand_i):
+          with ops.colocate_with(cand_i):
+            return array_ops.where(elements_finished, cur_i, cand_i)
+        return nest.map_structure(copy_fn, current, candidate)
+
+      emit_output = _copy_some_through(zero_emit, emit_output)
+      next_state = _copy_some_through(state, next_state)
+
+      emit_ta = nest.map_structure(
+          lambda ta, emit: ta.write(time, emit), emit_ta, emit_output)
+
+      elements_finished = math_ops.logical_or(elements_finished, next_finished)
+
+      return (next_time, elements_finished, next_input,
+              emit_ta, next_state, loop_state)
+
+    returned = control_flow_ops.while_loop(
+        condition, body, loop_vars=[
+            time, elements_finished, next_input,
+            emit_ta, state, loop_state],
+        parallel_iterations=parallel_iterations,
+        swap_memory=swap_memory)
+
+    (emit_ta, final_state, final_loop_state) = returned[-3:]
+
+    if init_loop_state is None:
+      final_loop_state = None
+
+    return (emit_ta, final_state, final_loop_state)
+
+
+def static_rnn(cell,
+               inputs,
+               initial_state=None,
+               dtype=None,
+               sequence_length=None,
+               scope=None):
+  """Creates a recurrent neural network specified by RNNCell `cell`.
+
+  The simplest form of RNN network generated is:
+
+  ```python
+    state = cell.zero_state(...)
+    outputs = []
+    for input_ in inputs:
+      output, state = cell(input_, state)
+      outputs.append(output)
+    return (outputs, state)
+  ```
+  However, a few other options are available:
+
+  An initial state can be provided.
+  If the sequence_length vector is provided, dynamic calculation is performed.
+  This method of calculation does not compute the RNN steps past the maximum
+  sequence length of the minibatch (thus saving computational time),
+  and properly propagates the state at an example's sequence length
+  to the final state output.
+
+  The dynamic calculation performed is, at time `t` for batch row `b`,
+
+  ```python
+    (output, state)(b, t) =
+      (t >= sequence_length(b))
+        ? (zeros(cell.output_size), states(b, sequence_length(b) - 1))
+        : cell(input(b, t), state(b, t - 1))
+  ```
+
+  Args:
+    cell: An instance of RNNCell.
+    inputs: A length T list of inputs, each a `Tensor` of shape
+      `[batch_size, input_size]`, or a nested tuple of such elements.
+    initial_state: (optional) An initial state for the RNN.
+      If `cell.state_size` is an integer, this must be
+      a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`.
+      If `cell.state_size` is a tuple, this should be a tuple of
+      tensors having shapes `[batch_size, s] for s in cell.state_size`.
+    dtype: (optional) The data type for the initial state and expected output.
+      Required if initial_state is not provided or RNN state has a heterogeneous
+      dtype.
+    sequence_length: Specifies the length of each sequence in inputs.
+      An int32 or int64 vector (tensor) size `[batch_size]`, values in `[0, T)`.
+    scope: VariableScope for the created subgraph; defaults to "rnn".
+
+  Returns:
+    A pair (outputs, state) where:
+
+    - outputs is a length T list of outputs (one for each input), or a nested
+      tuple of such elements.
+    - state is the final state
+
+  Raises:
+    TypeError: If `cell` is not an instance of RNNCell.
+    ValueError: If `inputs` is `None` or an empty list, or if the input depth
+      (column size) cannot be inferred from inputs via shape inference.
+  """
+
+  if not _like_rnncell(cell):
+    raise TypeError("cell must be an instance of RNNCell")
+  if not nest.is_sequence(inputs):
+    raise TypeError("inputs must be a sequence")
+  if not inputs:
+    raise ValueError("inputs must not be empty")
+
+  outputs = []
+  # Create a new scope in which the caching device is either
+  # determined by the parent scope, or is set to place the cached
+  # Variable using the same placement as for the rest of the RNN.
+  with vs.variable_scope(scope or "rnn") as varscope:
+    if varscope.caching_device is None:
+      varscope.set_caching_device(lambda op: op.device)
+
+    # Obtain the first sequence of the input
+    first_input = inputs
+    while nest.is_sequence(first_input):
+      first_input = first_input[0]
+
+    # Temporarily avoid EmbeddingWrapper and seq2seq badness
+    # TODO(lukaszkaiser): remove EmbeddingWrapper
+    if first_input.get_shape().ndims != 1:
+
+      input_shape = first_input.get_shape().with_rank_at_least(2)
+      fixed_batch_size = input_shape[0]
+
+      flat_inputs = nest.flatten(inputs)
+      for flat_input in flat_inputs:
+        input_shape = flat_input.get_shape().with_rank_at_least(2)
+        batch_size, input_size = input_shape[0], input_shape[1:]
+        fixed_batch_size.merge_with(batch_size)
+        for i, size in enumerate(input_size):
+          if size.value is None:
+            raise ValueError(
+                "Input size (dimension %d of inputs) must be accessible via "
+                "shape inference, but saw value None." % i)
+    else:
+      fixed_batch_size = first_input.get_shape().with_rank_at_least(1)[0]
+
+    if fixed_batch_size.value:
+      batch_size = fixed_batch_size.value
+    else:
+      batch_size = array_ops.shape(first_input)[0]
+    if initial_state is not None:
+      state = initial_state
+    else:
+      if not dtype:
+        raise ValueError("If no initial_state is provided, "
+                         "dtype must be specified")
+      state = cell.zero_state(batch_size, dtype)
+
+    if sequence_length is not None:  # Prepare variables
+      sequence_length = ops.convert_to_tensor(
+          sequence_length, name="sequence_length")
+      if sequence_length.get_shape().ndims not in (None, 1):
+        raise ValueError(
+            "sequence_length must be a vector of length batch_size")
+
+      def _create_zero_output(output_size):
+        # convert int to TensorShape if necessary
+        size = _concat(batch_size, output_size)
+        output = array_ops.zeros(
+            array_ops.stack(size), _infer_state_dtype(dtype, state))
+        shape = _concat(fixed_batch_size.value, output_size, static=True)
+        output.set_shape(tensor_shape.TensorShape(shape))
+        return output
+
+      output_size = cell.output_size
+      flat_output_size = nest.flatten(output_size)
+      flat_zero_output = tuple(
+          _create_zero_output(size) for size in flat_output_size)
+      zero_output = nest.pack_sequence_as(
+          structure=output_size, flat_sequence=flat_zero_output)
+
+      sequence_length = math_ops.to_int32(sequence_length)
+      min_sequence_length = math_ops.reduce_min(sequence_length)
+      max_sequence_length = math_ops.reduce_max(sequence_length)
+
+    for time, input_ in enumerate(inputs):
+      if time > 0:
+        varscope.reuse_variables()
+      # pylint: disable=cell-var-from-loop
+      call_cell = lambda: cell(input_, state)
+      # pylint: enable=cell-var-from-loop
+      if sequence_length is not None:
+        (output, state) = _rnn_step(
+            time=time,
+            sequence_length=sequence_length,
+            min_sequence_length=min_sequence_length,
+            max_sequence_length=max_sequence_length,
+            zero_output=zero_output,
+            state=state,
+            call_cell=call_cell,
+            state_size=cell.state_size)
+      else:
+        (output, state) = call_cell()
+
+      outputs.append(output)
+
+    return (outputs, state)
+
+
+def static_state_saving_rnn(cell,
+                            inputs,
+                            state_saver,
+                            state_name,
+                            sequence_length=None,
+                            scope=None):
+  """RNN that accepts a state saver for time-truncated RNN calculation.
+
+  Args:
+    cell: An instance of `RNNCell`.
+    inputs: A length T list of inputs, each a `Tensor` of shape
+      `[batch_size, input_size]`.
+    state_saver: A state saver object with methods `state` and `save_state`.
+    state_name: Python string or tuple of strings.  The name to use with the
+      state_saver. If the cell returns tuples of states (i.e.,
+      `cell.state_size` is a tuple) then `state_name` should be a tuple of
+      strings having the same length as `cell.state_size`.  Otherwise it should
+      be a single string.
+    sequence_length: (optional) An int32/int64 vector size [batch_size].
+      See the documentation for rnn() for more details about sequence_length.
+    scope: VariableScope for the created subgraph; defaults to "rnn".
+
+  Returns:
+    A pair (outputs, state) where:
+      outputs is a length T list of outputs (one for each input)
+      states is the final state
+
+  Raises:
+    TypeError: If `cell` is not an instance of RNNCell.
+    ValueError: If `inputs` is `None` or an empty list, or if the arity and
+     type of `state_name` does not match that of `cell.state_size`.
+  """
+  state_size = cell.state_size
+  state_is_tuple = nest.is_sequence(state_size)
+  state_name_tuple = nest.is_sequence(state_name)
+
+  if state_is_tuple != state_name_tuple:
+    raise ValueError("state_name should be the same type as cell.state_size.  "
+                     "state_name: %s, cell.state_size: %s" % (str(state_name),
+                                                              str(state_size)))
+
+  if state_is_tuple:
+    state_name_flat = nest.flatten(state_name)
+    state_size_flat = nest.flatten(state_size)
+
+    if len(state_name_flat) != len(state_size_flat):
+      raise ValueError("#elems(state_name) != #elems(state_size): %d vs. %d" %
+                       (len(state_name_flat), len(state_size_flat)))
+
+    initial_state = nest.pack_sequence_as(
+        structure=state_size,
+        flat_sequence=[state_saver.state(s) for s in state_name_flat])
+  else:
+    initial_state = state_saver.state(state_name)
+
+  (outputs, state) = static_rnn(
+      cell,
+      inputs,
+      initial_state=initial_state,
+      sequence_length=sequence_length,
+      scope=scope)
+
+  if state_is_tuple:
+    flat_state = nest.flatten(state)
+    state_name = nest.flatten(state_name)
+    save_state = [
+        state_saver.save_state(name, substate)
+        for name, substate in zip(state_name, flat_state)
+    ]
+  else:
+    save_state = [state_saver.save_state(state_name, state)]
+
+  with ops.control_dependencies(save_state):
+    last_output = outputs[-1]
+    flat_last_output = nest.flatten(last_output)
+    flat_last_output = [
+        array_ops.identity(output) for output in flat_last_output
+    ]
+    outputs[-1] = nest.pack_sequence_as(
+        structure=last_output, flat_sequence=flat_last_output)
+
+  return (outputs, state)
+
+
+def static_bidirectional_rnn(cell_fw,
+                             cell_bw,
+                             inputs,
+                             initial_state_fw=None,
+                             initial_state_bw=None,
+                             dtype=None,
+                             sequence_length=None,
+                             scope=None):
+  """Creates a bidirectional recurrent neural network.
+
+  Similar to the unidirectional case above (rnn) but takes input and builds
+  independent forward and backward RNNs with the final forward and backward
+  outputs depth-concatenated, such that the output will have the format
+  [time][batch][cell_fw.output_size + cell_bw.output_size]. The input_size of
+  forward and backward cell must match. The initial state for both directions
+  is zero by default (but can be set optionally) and no intermediate states are
+  ever returned -- the network is fully unrolled for the given (passed in)
+  length(s) of the sequence(s) or completely unrolled if length(s) is not given.
+
+  Args:
+    cell_fw: An instance of RNNCell, to be used for forward direction.
+    cell_bw: An instance of RNNCell, to be used for backward direction.
+    inputs: A length T list of inputs, each a tensor of shape
+      [batch_size, input_size], or a nested tuple of such elements.
+    initial_state_fw: (optional) An initial state for the forward RNN.
+      This must be a tensor of appropriate type and shape
+      `[batch_size, cell_fw.state_size]`.
+      If `cell_fw.state_size` is a tuple, this should be a tuple of
+      tensors having shapes `[batch_size, s] for s in cell_fw.state_size`.
+    initial_state_bw: (optional) Same as for `initial_state_fw`, but using
+      the corresponding properties of `cell_bw`.
+    dtype: (optional) The data type for the initial state.  Required if
+      either of the initial states are not provided.
+    sequence_length: (optional) An int32/int64 vector, size `[batch_size]`,
+      containing the actual lengths for each of the sequences.
+    scope: VariableScope for the created subgraph; defaults to
+      "bidirectional_rnn"
+
+  Returns:
+    A tuple (outputs, output_state_fw, output_state_bw) where:
+      outputs is a length `T` list of outputs (one for each input), which
+        are depth-concatenated forward and backward outputs.
+      output_state_fw is the final state of the forward rnn.
+      output_state_bw is the final state of the backward rnn.
+
+  Raises:
+    TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`.
+    ValueError: If inputs is None or an empty list.
+  """
+
+  if not _like_rnncell(cell_fw):
+    raise TypeError("cell_fw must be an instance of RNNCell")
+  if not _like_rnncell(cell_bw):
+    raise TypeError("cell_bw must be an instance of RNNCell")
+  if not nest.is_sequence(inputs):
+    raise TypeError("inputs must be a sequence")
+  if not inputs:
+    raise ValueError("inputs must not be empty")
+
+  with vs.variable_scope(scope or "bidirectional_rnn"):
+    # Forward direction
+    with vs.variable_scope("fw") as fw_scope:
+      output_fw, output_state_fw = static_rnn(
+          cell_fw,
+          inputs,
+          initial_state_fw,
+          dtype,
+          sequence_length,
+          scope=fw_scope)
+
+    # Backward direction
+    with vs.variable_scope("bw") as bw_scope:
+      reversed_inputs = _reverse_seq(inputs, sequence_length)
+      tmp, output_state_bw = static_rnn(
+          cell_bw,
+          reversed_inputs,
+          initial_state_bw,
+          dtype,
+          sequence_length,
+          scope=bw_scope)
+
+  output_bw = _reverse_seq(tmp, sequence_length)
+  # Concat each of the forward/backward outputs
+  flat_output_fw = nest.flatten(output_fw)
+  flat_output_bw = nest.flatten(output_bw)
+
+  flat_outputs = tuple(
+      array_ops.concat([fw, bw], 1)
+      for fw, bw in zip(flat_output_fw, flat_output_bw))
+
+  outputs = nest.pack_sequence_as(
+      structure=output_fw, flat_sequence=flat_outputs)
+
+  return (outputs, output_state_fw, output_state_bw)
diff --git a/recommendation/Basic-DIEN-Demo/source_code/shuffle.py b/recommendation/Basic-DIEN-Demo/source_code/shuffle.py
new file mode 100755
index 00000000..cbea6703
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/shuffle.py
@@ -0,0 +1,42 @@
+import os
+import sys
+import random
+
+import tempfile
+from subprocess import call
+
+
+def main(file, temporary=False):
+    tf_os, tpath = tempfile.mkstemp(dir='/home/mouna.mn/code/DIN-V2-CODE')
+    tf = open(tpath, 'w')
+
+    fd = open(file, "r")
+    for l in fd:
+        print >> tf, l.strip("\n")
+    tf.close()
+
+    lines = open(tpath, 'r').readlines()
+    random.shuffle(lines)
+    if temporary:
+        path, filename = os.path.split(os.path.realpath(file))
+        fd = tempfile.TemporaryFile(prefix=filename + '.shuf', dir=path)
+    else:
+        fd = open(file + '.shuf', 'w')
+
+    for l in lines:
+        s = l.strip("\n")
+        print >> fd, s
+
+    if temporary:
+        fd.seek(0)
+    else:
+        fd.close()
+
+    os.remove(tpath)
+
+    return fd
+
+
+if __name__ == '__main__':
+    main(sys.argv[1])
+
diff --git a/recommendation/Basic-DIEN-Demo/source_code/split_by_user.py b/recommendation/Basic-DIEN-Demo/source_code/split_by_user.py
new file mode 100755
index 00000000..cfa378a5
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/split_by_user.py
@@ -0,0 +1,20 @@
+import random
+
+fi = open("local_test", "r")
+ftrain = open("local_train_splitByUser", "w")
+ftest = open("local_test_splitByUser", "w")
+
+while True:
+    rand_int = random.randint(1, 10)
+    noclk_line = fi.readline().strip()
+    clk_line = fi.readline().strip()
+    if noclk_line == "" or clk_line == "":
+        break
+    if rand_int == 2:
+        print >> ftest, noclk_line
+        print >> ftest, clk_line
+    else:
+        print >> ftrain, noclk_line
+        print >> ftrain, clk_line
+        
+
diff --git a/recommendation/Basic-DIEN-Demo/source_code/train.py b/recommendation/Basic-DIEN-Demo/source_code/train.py
new file mode 100755
index 00000000..0db15546
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/train.py
@@ -0,0 +1,217 @@
+import numpy
+from data_iterator import DataIterator
+import tensorflow as tf
+from model import *
+import time
+import random
+import sys
+from utils import *
+
+EMBEDDING_DIM = 18
+HIDDEN_SIZE = 18 * 2
+ATTENTION_SIZE = 18 * 2
+best_auc = 0.0
+
+def prepare_data(input, target, maxlen = None, return_neg = False):
+    # x: a list of sentences
+    lengths_x = [len(s[4]) for s in input]
+    seqs_mid = [inp[3] for inp in input]
+    seqs_cat = [inp[4] for inp in input]
+    noclk_seqs_mid = [inp[5] for inp in input]
+    noclk_seqs_cat = [inp[6] for inp in input]
+
+    if maxlen is not None:
+        new_seqs_mid = []
+        new_seqs_cat = []
+        new_noclk_seqs_mid = []
+        new_noclk_seqs_cat = []
+        new_lengths_x = []
+
+
+    n_samples = len(seqs_mid)
+    maxlen_x = numpy.max(lengths_x)
+    neg_samples = len(noclk_seqs_mid[0][0])
+
+    mid_his = numpy.zeros((n_samples, maxlen_x)).astype('int64')
+    cat_his = numpy.zeros((n_samples, maxlen_x)).astype('int64')
+    noclk_mid_his = numpy.zeros((n_samples, maxlen_x, neg_samples)).astype('int64')
+    noclk_cat_his = numpy.zeros((n_samples, maxlen_x, neg_samples)).astype('int64')
+    mid_mask = numpy.zeros((n_samples, maxlen_x)).astype('float32')
+    for idx, [s_x, s_y, no_sx, no_sy] in enumerate(zip(seqs_mid, seqs_cat, noclk_seqs_mid, noclk_seqs_cat)):
+        mid_mask[idx, :lengths_x[idx]] = 1.
+        mid_his[idx, :lengths_x[idx]] = s_x
+        cat_his[idx, :lengths_x[idx]] = s_y
+        noclk_mid_his[idx, :lengths_x[idx], :] = no_sx
+        noclk_cat_his[idx, :lengths_x[idx], :] = no_sy
+
+    uids = numpy.array([inp[0] for inp in input])
+    mids = numpy.array([inp[1] for inp in input])
+    cats = numpy.array([inp[2] for inp in input])
+
+    if return_neg:
+        return uids, mids, cats, mid_his, cat_his, mid_mask, numpy.array(target), numpy.array(lengths_x), noclk_mid_his, noclk_cat_his
+
+    else:
+        return uids, mids, cats, mid_his, cat_his, mid_mask, numpy.array(target), numpy.array(lengths_x)
+
+def eval(sess, test_data, model, model_path):
+
+    loss_sum = 0.
+    accuracy_sum = 0.
+    aux_loss_sum = 0.
+    nums = 0
+    stored_arr = []
+    for src, tgt in test_data:
+        nums += 1
+        uids, mids, cats, mid_his, cat_his, mid_mask, target, sl, noclk_mids, noclk_cats = prepare_data(src, tgt, return_neg=True)
+        prob, loss, acc, aux_loss = model.calculate(sess, [uids, mids, cats, mid_his, cat_his, mid_mask, target, sl, noclk_mids, noclk_cats])
+        loss_sum += loss
+        aux_loss_sum = aux_loss
+        accuracy_sum += acc
+        prob_1 = prob[:, 0].tolist()
+        target_1 = target[:, 0].tolist()
+        for p ,t in zip(prob_1, target_1):
+            stored_arr.append([p, t])
+    test_auc = calc_auc(stored_arr)
+    accuracy_sum = accuracy_sum / nums
+    loss_sum = loss_sum / nums
+    aux_loss_sum / nums
+    global best_auc
+    if best_auc < test_auc:
+        best_auc = test_auc
+        model.save(sess, model_path)
+    return test_auc, loss_sum, accuracy_sum, aux_loss_sum
+
+def train(
+        train_file = "local_train_splitByUser",
+        test_file = "local_test_splitByUser",
+        uid_voc = "uid_voc.pkl",
+        mid_voc = "mid_voc.pkl",
+        cat_voc = "cat_voc.pkl",
+        batch_size = 128,
+        maxlen = 100,
+        test_iter = 100,
+        save_iter = 100,
+        model_type = 'DNN',
+	seed = 2,
+):
+    model_path = "dnn_save_path/ckpt_noshuff" + model_type + str(seed)
+    best_model_path = "dnn_best_model/ckpt_noshuff" + model_type + str(seed)
+    gpu_options = tf.GPUOptions(allow_growth=True)
+    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
+        train_data = DataIterator(train_file, uid_voc, mid_voc, cat_voc, batch_size, maxlen, shuffle_each_epoch=False)
+        test_data = DataIterator(test_file, uid_voc, mid_voc, cat_voc, batch_size, maxlen)
+        n_uid, n_mid, n_cat = train_data.get_n()
+        if model_type == 'DNN':
+            model = Model_DNN(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'PNN':
+            model = Model_PNN(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+	elif model_type == 'Wide':
+            model = Model_WideDeep(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN':
+            model = Model_DIN(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN-V2-gru-att-gru':
+            model = Model_DIN_V2_Gru_att_Gru(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN-V2-gru-gru-att':
+            model = Model_DIN_V2_Gru_Gru_att(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN-V2-gru-qa-attGru':
+            model = Model_DIN_V2_Gru_QA_attGru(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN-V2-gru-vec-attGru':
+            model = Model_DIN_V2_Gru_Vec_attGru(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+	elif model_type == 'DIEN':
+            model = Model_DIN_V2_Gru_Vec_attGru_Neg(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        else:
+            print ("Invalid model_type : %s", model_type)
+            return
+        # model = Model_DNN(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        sess.run(tf.global_variables_initializer())
+        sess.run(tf.local_variables_initializer())
+        sys.stdout.flush()
+        print('                                                                                      test_auc: %.4f ---- test_loss: %.4f ---- test_accuracy: %.4f ---- test_aux_loss: %.4f' % eval(sess, test_data, model, best_model_path))
+        sys.stdout.flush()
+
+        start_time = time.time()
+        iter = 0
+        lr = 0.001
+        for itr in range(3):
+            loss_sum = 0.0
+            accuracy_sum = 0.
+            aux_loss_sum = 0.
+            for src, tgt in train_data:
+                uids, mids, cats, mid_his, cat_his, mid_mask, target, sl, noclk_mids, noclk_cats = prepare_data(src, tgt, maxlen, return_neg=True)
+                loss, acc, aux_loss = model.train(sess, [uids, mids, cats, mid_his, cat_his, mid_mask, target, sl, lr, noclk_mids, noclk_cats])
+                loss_sum += loss
+                accuracy_sum += acc
+                aux_loss_sum += aux_loss
+                iter += 1
+                sys.stdout.flush()
+                if (iter % test_iter) == 0:
+                    print('iter: %d ----> train_loss: %.4f ---- train_accuracy: %.4f ---- tran_aux_loss: %.4f' % \
+                                          (iter, loss_sum / test_iter, accuracy_sum / test_iter, aux_loss_sum / test_iter))
+                    print('                                                                                          test_auc: %.4f ----test_loss: %.4f ---- test_accuracy: %.4f ---- test_aux_loss: %.4f' % eval(sess, test_data, model, best_model_path))
+                    loss_sum = 0.0
+                    accuracy_sum = 0.0
+                    aux_loss_sum = 0.0
+                if (iter % save_iter) == 0:
+                    print('save model iter: %d' %(iter))
+                    model.save(sess, model_path+"--"+str(iter))
+            lr *= 0.5
+
+def test(
+        train_file = "local_train_splitByUser",
+        test_file = "local_test_splitByUser",
+        uid_voc = "uid_voc.pkl",
+        mid_voc = "mid_voc.pkl",
+        cat_voc = "cat_voc.pkl",
+        batch_size = 128,
+        maxlen = 100,
+        model_type = 'DNN',
+	seed = 2
+):
+
+    model_path = "dnn_best_model/ckpt_noshuff" + model_type + str(seed)
+    gpu_options = tf.GPUOptions(allow_growth=True)
+    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
+        train_data = DataIterator(train_file, uid_voc, mid_voc, cat_voc, batch_size, maxlen)
+        test_data = DataIterator(test_file, uid_voc, mid_voc, cat_voc, batch_size, maxlen)
+        n_uid, n_mid, n_cat = train_data.get_n()
+        if model_type == 'DNN':
+            model = Model_DNN(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'PNN':
+            model = Model_PNN(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'Wide':
+	    model = Model_WideDeep(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN':
+            model = Model_DIN(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN-V2-gru-att-gru':
+            model = Model_DIN_V2_Gru_att_Gru(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN-V2-gru-gru-att':
+            model = Model_DIN_V2_Gru_Gru_att(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN-V2-gru-qa-attGru':
+            model = Model_DIN_V2_Gru_QA_attGru(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        elif model_type == 'DIN-V2-gru-vec-attGru':
+            model = Model_DIN_V2_Gru_Vec_attGru(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+	elif model_type == 'DIEN':
+            model = Model_DIN_V2_Gru_Vec_attGru_Neg(n_uid, n_mid, n_cat, EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+        else:
+            print ("Invalid model_type : %s", model_type)
+            return
+        model.restore(sess, model_path)
+        print('test_auc: %.4f ----test_loss: %.4f ---- test_accuracy: %.4f ---- test_aux_loss: %.4f' % eval(sess, test_data, model, model_path))
+
+if __name__ == '__main__':
+    if len(sys.argv) == 4:
+        SEED = int(sys.argv[3])
+    else:
+        SEED = 3
+    tf.set_random_seed(SEED)
+    numpy.random.seed(SEED)
+    random.seed(SEED)
+    if sys.argv[1] == 'train':
+        train(model_type=sys.argv[2], seed=SEED)
+    elif sys.argv[1] == 'test':
+        test(model_type=sys.argv[2], seed=SEED)
+    else:
+        print('do nothing...')
+
+
diff --git a/recommendation/Basic-DIEN-Demo/source_code/utils.py b/recommendation/Basic-DIEN-Demo/source_code/utils.py
new file mode 100755
index 00000000..fe76b56e
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/source_code/utils.py
@@ -0,0 +1,403 @@
+import tensorflow as tf
+from tensorflow.python.ops.rnn_cell import *
+from tensorflow.python.ops.rnn_cell_impl import  _Linear
+
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variable_scope as vs
+
+
+class QAAttGRUCell(RNNCell):
+  """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
+  Args:
+    num_units: int, The number of units in the GRU cell.
+    activation: Nonlinearity to use.  Default: `tanh`.
+    reuse: (optional) Python boolean describing whether to reuse variables
+     in an existing scope.  If not `True`, and the existing scope already has
+     the given variables, an error is raised.
+    kernel_initializer: (optional) The initializer to use for the weight and
+    projection matrices.
+    bias_initializer: (optional) The initializer to use for the bias.
+  """
+
+  def __init__(self,
+               num_units,
+               activation=None,
+               reuse=None,
+               kernel_initializer=None,
+               bias_initializer=None):
+    super(QAAttGRUCell, self).__init__(_reuse=reuse)
+    self._num_units = num_units
+    self._activation = activation or math_ops.tanh
+    self._kernel_initializer = kernel_initializer
+    self._bias_initializer = bias_initializer
+    self._gate_linear = None
+    self._candidate_linear = None
+
+  @property
+  def state_size(self):
+    return self._num_units
+
+  @property
+  def output_size(self):
+    return self._num_units
+
+  def __call__(self, inputs, state, att_score):
+      return self.call(inputs, state, att_score)
+
+  def call(self, inputs, state, att_score=None):
+    """Gated recurrent unit (GRU) with nunits cells."""
+    if self._gate_linear is None:
+      bias_ones = self._bias_initializer
+      if self._bias_initializer is None:
+        bias_ones = init_ops.constant_initializer(1.0, dtype=inputs.dtype)
+      with vs.variable_scope("gates"):  # Reset gate and update gate.
+        self._gate_linear = _Linear(
+            [inputs, state],
+            2 * self._num_units,
+            True,
+            bias_initializer=bias_ones,
+            kernel_initializer=self._kernel_initializer)
+
+    value = math_ops.sigmoid(self._gate_linear([inputs, state]))
+    r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
+
+    r_state = r * state
+    if self._candidate_linear is None:
+      with vs.variable_scope("candidate"):
+        self._candidate_linear = _Linear(
+            [inputs, r_state],
+            self._num_units,
+            True,
+            bias_initializer=self._bias_initializer,
+            kernel_initializer=self._kernel_initializer)
+    c = self._activation(self._candidate_linear([inputs, r_state]))
+    new_h = (1. - att_score) * state + att_score * c
+    return new_h, new_h
+
+class VecAttGRUCell(RNNCell):
+  """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
+  Args:
+    num_units: int, The number of units in the GRU cell.
+    activation: Nonlinearity to use.  Default: `tanh`.
+    reuse: (optional) Python boolean describing whether to reuse variables
+     in an existing scope.  If not `True`, and the existing scope already has
+     the given variables, an error is raised.
+    kernel_initializer: (optional) The initializer to use for the weight and
+    projection matrices.
+    bias_initializer: (optional) The initializer to use for the bias.
+  """
+
+  def __init__(self,
+               num_units,
+               activation=None,
+               reuse=None,
+               kernel_initializer=None,
+               bias_initializer=None):
+    super(VecAttGRUCell, self).__init__(_reuse=reuse)
+    self._num_units = num_units
+    self._activation = activation or math_ops.tanh
+    self._kernel_initializer = kernel_initializer
+    self._bias_initializer = bias_initializer
+    self._gate_linear = None
+    self._candidate_linear = None
+
+  @property
+  def state_size(self):
+    return self._num_units
+
+  @property
+  def output_size(self):
+    return self._num_units
+  def __call__(self, inputs, state, att_score):
+      return self.call(inputs, state, att_score)
+  def call(self, inputs, state, att_score=None):
+    """Gated recurrent unit (GRU) with nunits cells."""
+    if self._gate_linear is None:
+      bias_ones = self._bias_initializer
+      if self._bias_initializer is None:
+        bias_ones = init_ops.constant_initializer(1.0, dtype=inputs.dtype)
+      with vs.variable_scope("gates"):  # Reset gate and update gate.
+        self._gate_linear = _Linear(
+            [inputs, state],
+            2 * self._num_units,
+            True,
+            bias_initializer=bias_ones,
+            kernel_initializer=self._kernel_initializer)
+
+    value = math_ops.sigmoid(self._gate_linear([inputs, state]))
+    r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
+
+    r_state = r * state
+    if self._candidate_linear is None:
+      with vs.variable_scope("candidate"):
+        self._candidate_linear = _Linear(
+            [inputs, r_state],
+            self._num_units,
+            True,
+            bias_initializer=self._bias_initializer,
+            kernel_initializer=self._kernel_initializer)
+    c = self._activation(self._candidate_linear([inputs, r_state]))
+    u = (1.0 - att_score) * u
+    new_h = u * state + (1 - u) * c
+    return new_h, new_h
+
+def prelu(_x, scope=''):
+    """parametric ReLU activation"""
+    with tf.variable_scope(name_or_scope=scope, default_name="prelu"):
+        _alpha = tf.get_variable("prelu_"+scope, shape=_x.get_shape()[-1],
+                                 dtype=_x.dtype, initializer=tf.constant_initializer(0.1))
+        return tf.maximum(0.0, _x) + _alpha * tf.minimum(0.0, _x)
+
+def calc_auc(raw_arr):
+    """Summary
+
+    Args:
+        raw_arr (TYPE): Description
+
+    Returns:
+        TYPE: Description
+    """
+
+    arr = sorted(raw_arr, key=lambda d:d[0], reverse=True)
+    pos, neg = 0., 0.
+    for record in arr:
+        if record[1] == 1.:
+            pos += 1
+        else:
+            neg += 1
+
+    fp, tp = 0., 0.
+    xy_arr = []
+    for record in arr:
+        if record[1] == 1.:
+            tp += 1
+        else:
+            fp += 1
+        xy_arr.append([fp/neg, tp/pos])
+
+    auc = 0.
+    prev_x = 0.
+    prev_y = 0.
+    for x, y in xy_arr:
+        if x != prev_x:
+            auc += ((x - prev_x) * (y + prev_y) / 2.)
+            prev_x = x
+            prev_y = y
+
+    return auc
+
+def attention(query, facts, attention_size, mask, stag='null', mode='LIST', softmax_stag=1, time_major=False, return_alphas=False):
+    if isinstance(facts, tuple):
+        # In case of Bi-RNN, concatenate the forward and the backward RNN outputs.
+        facts = tf.concat(facts, 2)
+
+    if time_major:
+        # (T,B,D) => (B,T,D)
+        facts = tf.array_ops.transpose(facts, [1, 0, 2])
+
+    mask = tf.equal(mask, tf.ones_like(mask))
+    hidden_size = facts.get_shape().as_list()[-1]  # D value - hidden size of the RNN layer
+    input_size = query.get_shape().as_list()[-1]
+
+    # Trainable parameters
+    w1 = tf.Variable(tf.random_normal([hidden_size, attention_size], stddev=0.1))
+    w2 = tf.Variable(tf.random_normal([input_size, attention_size], stddev=0.1))
+    b = tf.Variable(tf.random_normal([attention_size], stddev=0.1))
+    v = tf.Variable(tf.random_normal([attention_size], stddev=0.1))
+
+    with tf.name_scope('v'):
+        # Applying fully connected layer with non-linear activation to each of the B*T timestamps;
+        #  the shape of `tmp` is (B,T,D)*(D,A)=(B,T,A), where A=attention_size
+        tmp1 = tf.tensordot(facts, w1, axes=1)
+        tmp2 = tf.tensordot(query, w2, axes=1)
+        tmp2 = tf.reshape(tmp2, [-1, 1, tf.shape(tmp2)[-1]])
+        tmp = tf.tanh((tmp1 + tmp2) + b)
+
+    # For each of the timestamps its vector of size A from `tmp` is reduced with `v` vector
+    v_dot_tmp = tf.tensordot(tmp, v, axes=1, name='v_dot_tmp')  # (B,T) shape
+    key_masks = mask # [B, 1, T]
+    # key_masks = tf.expand_dims(mask, 1) # [B, 1, T]
+    paddings = tf.ones_like(v_dot_tmp) * (-2 ** 32 + 1)
+    v_dot_tmp = tf.where(key_masks, v_dot_tmp, paddings)  # [B, 1, T]
+    alphas = tf.nn.softmax(v_dot_tmp, name='alphas')         # (B,T) shape
+
+    # Output of (Bi-)RNN is reduced with attention vector; the result has (B,D) shape
+    #output = tf.reduce_sum(facts * tf.expand_dims(alphas, -1), 1)
+    output = facts * tf.expand_dims(alphas, -1)
+    output = tf.reshape(output, tf.shape(facts))
+    # output = output / (facts.get_shape().as_list()[-1] ** 0.5)
+    if not return_alphas:
+        return output
+    else:
+        return output, alphas
+
+def din_attention(query, facts, attention_size, mask, stag='null', mode='SUM', softmax_stag=1, time_major=False, return_alphas=False):
+    if isinstance(facts, tuple):
+        # In case of Bi-RNN, concatenate the forward and the backward RNN outputs.
+        facts = tf.concat(facts, 2)
+        print ("querry_size mismatch")
+        query = tf.concat(values = [
+        query,
+        query,
+        ], axis=1)
+
+    if time_major:
+        # (T,B,D) => (B,T,D)
+        facts = tf.array_ops.transpose(facts, [1, 0, 2])
+    mask = tf.equal(mask, tf.ones_like(mask))
+    facts_size = facts.get_shape().as_list()[-1]  # D value - hidden size of the RNN layer
+    querry_size = query.get_shape().as_list()[-1]
+    queries = tf.tile(query, [1, tf.shape(facts)[1]])
+    queries = tf.reshape(queries, tf.shape(facts))
+    din_all = tf.concat([queries, facts, queries-facts, queries*facts], axis=-1)
+    d_layer_1_all = tf.layers.dense(din_all, 80, activation=tf.nn.sigmoid, name='f1_att' + stag)
+    d_layer_2_all = tf.layers.dense(d_layer_1_all, 40, activation=tf.nn.sigmoid, name='f2_att' + stag)
+    d_layer_3_all = tf.layers.dense(d_layer_2_all, 1, activation=None, name='f3_att' + stag)
+    d_layer_3_all = tf.reshape(d_layer_3_all, [-1, 1, tf.shape(facts)[1]])
+    scores = d_layer_3_all
+    # Mask
+    # key_masks = tf.sequence_mask(facts_length, tf.shape(facts)[1])   # [B, T]
+    key_masks = tf.expand_dims(mask, 1) # [B, 1, T]
+    paddings = tf.ones_like(scores) * (-2 ** 32 + 1)
+    scores = tf.where(key_masks, scores, paddings)  # [B, 1, T]
+
+    # Scale
+    # scores = scores / (facts.get_shape().as_list()[-1] ** 0.5)
+
+    # Activation
+    if softmax_stag:
+        scores = tf.nn.softmax(scores)  # [B, 1, T]
+
+    # Weighted sum
+    if mode == 'SUM':
+        output = tf.matmul(scores, facts)  # [B, 1, H]
+        # output = tf.reshape(output, [-1, tf.shape(facts)[-1]])
+    else:
+        scores = tf.reshape(scores, [-1, tf.shape(facts)[1]])
+        output = facts * tf.expand_dims(scores, -1)
+        output = tf.reshape(output, tf.shape(facts))
+    return output
+
+def din_fcn_attention(query, facts, attention_size, mask, stag='null', mode='SUM', softmax_stag=1, time_major=False, return_alphas=False, forCnn=False):
+    if isinstance(facts, tuple):
+        # In case of Bi-RNN, concatenate the forward and the backward RNN outputs.
+        facts = tf.concat(facts, 2)
+    if len(facts.get_shape().as_list()) == 2:
+        facts = tf.expand_dims(facts, 1)
+
+    if time_major:
+        # (T,B,D) => (B,T,D)
+        facts = tf.array_ops.transpose(facts, [1, 0, 2])
+    # Trainable parameters
+    mask = tf.equal(mask, tf.ones_like(mask))
+    facts_size = facts.get_shape().as_list()[-1]  # D value - hidden size of the RNN layer
+    querry_size = query.get_shape().as_list()[-1]
+    query = tf.layers.dense(query, facts_size, activation=None, name='f1' + stag)
+    query = prelu(query)
+    queries = tf.tile(query, [1, tf.shape(facts)[1]])
+    queries = tf.reshape(queries, tf.shape(facts))
+    din_all = tf.concat([queries, facts, queries-facts, queries*facts], axis=-1)
+    d_layer_1_all = tf.layers.dense(din_all, 80, activation=tf.nn.sigmoid, name='f1_att' + stag)
+    d_layer_2_all = tf.layers.dense(d_layer_1_all, 40, activation=tf.nn.sigmoid, name='f2_att' + stag)
+    d_layer_3_all = tf.layers.dense(d_layer_2_all, 1, activation=None, name='f3_att' + stag)
+    d_layer_3_all = tf.reshape(d_layer_3_all, [-1, 1, tf.shape(facts)[1]])
+    scores = d_layer_3_all
+    # Mask
+    # key_masks = tf.sequence_mask(facts_length, tf.shape(facts)[1])   # [B, T]
+    key_masks = tf.expand_dims(mask, 1) # [B, 1, T]
+    paddings = tf.ones_like(scores) * (-2 ** 32 + 1)
+    if not forCnn:
+        scores = tf.where(key_masks, scores, paddings)  # [B, 1, T]
+
+    # Scale
+    # scores = scores / (facts.get_shape().as_list()[-1] ** 0.5)
+
+    # Activation
+    if softmax_stag:
+        scores = tf.nn.softmax(scores)  # [B, 1, T]
+
+    # Weighted sum
+    if mode == 'SUM':
+        output = tf.matmul(scores, facts)  # [B, 1, H]
+        # output = tf.reshape(output, [-1, tf.shape(facts)[-1]])
+    else:
+        scores = tf.reshape(scores, [-1, tf.shape(facts)[1]])
+        output = facts * tf.expand_dims(scores, -1)
+        output = tf.reshape(output, tf.shape(facts))
+    if return_alphas:
+        return output, scores
+    return output
+
+def self_attention(facts, ATTENTION_SIZE, mask, stag='null'):
+    if len(facts.get_shape().as_list()) == 2:
+        facts = tf.expand_dims(facts, 1)
+
+    def cond(batch, output, i):
+        return tf.less(i, tf.shape(batch)[1])
+
+    def body(batch, output, i):
+        self_attention_tmp = din_fcn_attention(batch[:, i, :], batch[:, 0:i+1, :],
+                                               ATTENTION_SIZE, mask[:, 0:i+1], softmax_stag=1, stag=stag,
+                                               mode='LIST')
+        self_attention_tmp = tf.reduce_sum(self_attention_tmp, 1)
+        output = output.write(i, self_attention_tmp)
+        return batch, output, i + 1
+
+    output_ta = tf.TensorArray(dtype=tf.float32,
+                               size=0,
+                               dynamic_size=True,
+                               element_shape=(facts[:, 0, :].get_shape()))
+    _, output_op, _ = tf.while_loop(cond, body, [facts, output_ta, 0])
+    self_attention = output_op.stack()
+    self_attention = tf.transpose(self_attention, perm = [1, 0, 2])
+    return self_attention
+
+def self_all_attention(facts, ATTENTION_SIZE, mask, stag='null'):
+    if len(facts.get_shape().as_list()) == 2:
+        facts = tf.expand_dims(facts, 1)
+
+    def cond(batch, output, i):
+        return tf.less(i, tf.shape(batch)[1])
+
+    def body(batch, output, i):
+        self_attention_tmp = din_fcn_attention(batch[:, i, :], batch,
+                                               ATTENTION_SIZE, mask, softmax_stag=1, stag=stag,
+                                               mode='LIST')
+        self_attention_tmp = tf.reduce_sum(self_attention_tmp, 1)
+        output = output.write(i, self_attention_tmp)
+        return batch, output, i + 1
+
+    output_ta = tf.TensorArray(dtype=tf.float32,
+                               size=0,
+                               dynamic_size=True,
+                               element_shape=(facts[:, 0, :].get_shape()))
+    _, output_op, _ = tf.while_loop(cond, body, [facts, output_ta, 0])
+    self_attention = output_op.stack()
+    self_attention = tf.transpose(self_attention, perm = [1, 0, 2])
+    return self_attention
+
+def din_fcn_shine(query, facts, attention_size, mask, stag='null', mode='SUM', softmax_stag=1, time_major=False, return_alphas=False):
+    if isinstance(facts, tuple):
+        # In case of Bi-RNN, concatenate the forward and the backward RNN outputs.
+        facts = tf.concat(facts, 2)
+
+    if time_major:
+        # (T,B,D) => (B,T,D)
+        facts = tf.array_ops.transpose(facts, [1, 0, 2])
+    # Trainable parameters
+    mask = tf.equal(mask, tf.ones_like(mask))
+    facts_size = facts.get_shape().as_list()[-1]  # D value - hidden size of the RNN layer
+    querry_size = query.get_shape().as_list()[-1]
+    query = tf.layers.dense(query, facts_size, activation=None, name='f1_trans_shine' + stag)
+    query = prelu(query)
+    queries = tf.tile(query, [1, tf.shape(facts)[1]])
+    queries = tf.reshape(queries, tf.shape(facts))
+    din_all = tf.concat([queries, facts, queries-facts, queries*facts], axis=-1)
+    d_layer_1_all = tf.layers.dense(din_all, facts_size, activation=tf.nn.sigmoid, name='f1_shine_att' + stag)
+    d_layer_2_all = tf.layers.dense(d_layer_1_all, facts_size, activation=tf.nn.sigmoid, name='f2_shine_att' + stag)
+    d_layer_2_all = tf.reshape(d_layer_2_all, tf.shape(facts))
+    output = d_layer_2_all
+    return output
+
diff --git a/recommendation/Basic-DIEN-Demo/train.py b/recommendation/Basic-DIEN-Demo/train.py
new file mode 100644
index 00000000..c46a2e32
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/train.py
@@ -0,0 +1,138 @@
+import numpy as np
+from data_iterator import DataIterator
+import pandas as pd
+import tensorflow as tf
+import random
+from model import Model_DIN_V2_Gru_Vec_attGru_Neg
+
+
+
+EMBEDDING_DIM = 18
+HIDDEN_SIZE = 18 * 2
+ATTENTION_SIZE = 18 * 2
+best_auc = 0.0
+
+def prepare_data(input,target,maxlen = None,return_neg=False):
+    lengths_x = [len(s[4]) for s in input]
+    seqs_mid = [inp[3] for inp in input]
+    seqs_cat = [inp[4] for inp in input]
+    noclk_seqs_mid = [inp[5] for inp in input]
+    noclk_seqs_cat = [inp[6] for inp in input]
+
+    # 如果历史记录的长度超过设定的maxlen，则进行裁剪
+    if maxlen is not None:
+        new_seqs_mid = []
+        new_seqs_cat = []
+        new_noclk_seqs_mid = []
+        new_noclk_seqs_cat = []
+        new_lengths_x = []
+
+        for l_x, inp in zip(lengths_x, input):
+            if l_x > maxlen:
+                new_seqs_mid.append(inp[3][l_x - maxlen:])
+                new_seqs_cat.append(inp[4][l_x - maxlen:])
+                new_noclk_seqs_mid.append(inp[5][l_x - maxlen:])
+                new_noclk_seqs_cat.append(inp[6][l_x - maxlen:])
+                new_lengths_x.append(maxlen)
+            else:
+                new_seqs_mid.append(inp[3])
+                new_seqs_cat.append(inp[4])
+                new_noclk_seqs_mid.append(inp[5])
+                new_noclk_seqs_cat.append(inp[6])
+                new_lengths_x.append(l_x)
+        lengths_x = new_lengths_x
+        seqs_mid = new_seqs_mid
+        seqs_cat = new_seqs_cat
+        noclk_seqs_mid = new_noclk_seqs_mid
+        noclk_seqs_cat = new_noclk_seqs_cat
+
+        if len(lengths_x) < 1:
+            return None, None, None, None
+
+    # 进一步压缩空间，根据
+    n_samples = len(seqs_mid)
+    max_len_x = np.max(lengths_x)
+    neg_samples = len(noclk_seqs_mid[0][0])
+
+    mid_his = np.zeros((n_samples,max_len_x)).astype("int64")
+    cat_his = np.zeros((n_samples,max_len_x)).astype("int64")
+    noclk_mid_his = np.zeros((n_samples,max_len_x,neg_samples)).astype("int64")
+    noclk_cat_his =  np.zeros((n_samples,max_len_x,neg_samples)).astype("int64")
+    mid_mask = np.zeros((n_samples,max_len_x)).astype('float32')
+
+    for idx, [s_x, s_y, no_sx, no_sy] in enumerate(zip(seqs_mid, seqs_cat, noclk_seqs_mid, noclk_seqs_cat)):
+        mid_mask[idx, :lengths_x[idx]] = 1.
+        mid_his[idx, :lengths_x[idx]] = s_x
+        cat_his[idx, :lengths_x[idx]] = s_y
+        noclk_mid_his[idx, :lengths_x[idx], :] = no_sx
+        noclk_cat_his[idx, :lengths_x[idx], :] = no_sy
+
+    uids = np.array([inp[0] for inp in input])
+    mids = np.array([inp[1] for inp in input])
+    cats = np.array([inp[2] for inp in input])
+
+    if return_neg:
+        return uids, mids, cats, mid_his, cat_his, mid_mask, np.array(target), np.array(lengths_x), noclk_mid_his, noclk_cat_his
+
+    else:
+        return uids, mids, cats, mid_his, cat_his, mid_mask, np.array(target), np.array(lengths_x)
+
+
+
+def train(train_file = "data/local_train_splitByUser",
+        test_file = "data/local_test_splitByUser",
+        uid_voc = "data/uid_voc.pkl",
+        mid_voc = "data/mid_voc.pkl",
+        cat_voc = "data/cat_voc.pkl",
+        batch_size = 128,
+        maxlen = 100,
+        test_iter = 100,
+        save_iter = 100,
+        model_type = 'DNN',
+	    seed = 2,
+    ):
+
+    model_path = "dnn_save_path/ckpt_noshuff" + model_type + str(seed)
+    best_model_path = "dnn_bast_model/ckpt_noshuff" + model_type + str(seed)
+    gpu_options = tf.GPUOptions(allow_growth=True)
+    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
+        train_data = DataIterator(train_file,uid_voc,mid_voc,cat_voc,batch_size,maxlen)
+        test_data = DataIterator(test_file,uid_voc,mid_voc,cat_voc,batch_size,maxlen)
+
+        n_uid,n_mid,n_cat = train_data.get_n()
+        model = Model_DIN_V2_Gru_Vec_attGru_Neg(n_uid,n_mid,n_cat,EMBEDDING_DIM, HIDDEN_SIZE, ATTENTION_SIZE)
+
+        sess.run(tf.global_variables_initializer())
+        sess.run(tf.local_variables_initializer())
+        iter = 0
+        lr = 0.001
+        for itr in range(3):
+            loss_sum = 0.0
+            accuracy_sum = 0.0
+            aux_loss_sum = 0.0
+
+            for src,tgt in train_data:
+                uids, mids, cats, mid_his, cat_his, mid_mask, target, sl, noclk_mids, noclk_cats = prepare_data(src,tgt,maxlen,return_neg=True)
+                loss, acc, aux_loss = model.train(sess, [uids, mids, cats, mid_his, cat_his, mid_mask, target, sl, lr,noclk_mids, noclk_cats])
+                loss_sum += loss
+                accuracy_sum += acc
+                aux_loss_sum += aux_loss
+                iter += 1
+                if (iter % test_iter) == 0:
+                    print('iter: %d ----> train_loss: %.8f ---- train_accuracy: %.4f ---- tran_aux_loss: %.4f' % \
+                          (iter, loss_sum / test_iter, accuracy_sum / test_iter, aux_loss_sum / test_iter))
+                    loss_sum = 0.0
+                    accuracy_sum = 0.0
+                    aux_loss_sum = 0.0
+                if (iter % save_iter) == 0:
+                    print('save model iter: %d' % (iter))
+                    model.save(sess, model_path + "--" + str(iter))
+            lr *= 0.5
+
+
+if __name__ == '__main__':
+    SEED = 5
+    tf.set_random_seed(SEED)
+    np.random.seed(SEED)
+    random.seed(SEED)
+    train()
\ No newline at end of file
diff --git a/recommendation/Basic-DIEN-Demo/utils.py b/recommendation/Basic-DIEN-Demo/utils.py
new file mode 100644
index 00000000..5f13e3e9
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/utils.py
@@ -0,0 +1,66 @@
+import tensorflow as tf
+
+def prelu(_x, scope=''):
+    """parametric ReLU activation"""
+    with tf.variable_scope(name_or_scope=scope, default_name="prelu"):
+        _alpha = tf.get_variable("prelu_"+scope, shape=_x.get_shape()[-1],
+                                 dtype=_x.dtype, initializer=tf.constant_initializer(0.1))
+        return tf.maximum(0.0, _x) + _alpha * tf.minimum(0.0, _x)
+
+
+def din_fcn_attention(query, facts, attention_size, mask, stag='null', mode='SUM', softmax_stag=1, time_major=False, return_alphas=False, forCnn=False):
+    if isinstance(facts, tuple):
+        # In case of Bi-RNN, concatenate the forward and the backward RNN outputs.
+        facts = tf.concat(facts, 2)
+    if len(facts.get_shape().as_list()) == 2:
+        facts = tf.expand_dims(facts, 1)
+
+    if time_major:
+        # (T,B,D) => (B,T,D)
+        facts = tf.array_ops.transpose(facts, [1, 0, 2])
+
+
+    mask = tf.equal(mask,tf.ones_like(mask))
+    facts_size = facts.get_shape().as_list()[-1] # Hidden size for rnn layer
+    query = tf.layers.dense(query,facts_size,activation=None,name='f1'+stag)
+    query = prelu(query)
+
+    queries = tf.tile(query,[1,tf.shape(facts)[1]]) # Batch * Time * Hidden size
+    queries = tf.reshape(queries,tf.shape(facts))
+    din_all = tf.concat([queries,facts,queries-facts,queries*facts],axis=-1) # Batch * Time * (4 * Hidden size)
+    d_layer_1_all = tf.layers.dense(din_all, 80, activation=tf.nn.sigmoid, name='f1_att' + stag)
+    d_layer_2_all = tf.layers.dense(d_layer_1_all, 40, activation=tf.nn.sigmoid, name='f2_att' + stag)
+    d_layer_3_all = tf.layers.dense(d_layer_2_all, 1, activation=None, name='f3_att' + stag) # Batch * Time * 1
+
+    d_layer_3_all = tf.reshape(d_layer_3_all,[-1,1,tf.shape(facts)[1]])  # Batch * 1 * time
+    scores = d_layer_3_all
+
+    key_masks = tf.expand_dims(mask,1) # Batch * 1 * Time
+    paddings = tf.ones_like(scores) * (-2 ** 32 + 1)
+
+    if not forCnn:
+        scores = tf.where(key_masks, scores, paddings)  # [B, 1, T] ，没有的地方用paddings填充
+
+    # Activation
+    if softmax_stag:
+        scores = tf.nn.softmax(scores)  # [B, 1, T]
+
+    # Weighted sum
+    if mode == 'SUM':
+        output = tf.matmul(scores,facts) # Batch * 1 * Hidden Size
+    else:
+        scores = tf.reshape(scores,[-1,tf.shape(facts)[1]]) # Batch * Time
+        output = facts * tf.expand_dims(scores,-1) # Batch * Time * Hidden Size
+        output = tf.reshape(output,tf.shape(facts))
+    if return_alphas:
+        return output,scores
+    else:
+        return output
+
+
+
+
+
+
+
+
diff --git a/recommendation/Basic-DIEN-Demo/vecAttGruCell.py b/recommendation/Basic-DIEN-Demo/vecAttGruCell.py
new file mode 100644
index 00000000..b7ec86d0
--- /dev/null
+++ b/recommendation/Basic-DIEN-Demo/vecAttGruCell.py
@@ -0,0 +1,68 @@
+import tensorflow as tf
+from tensorflow.python.ops.rnn_cell import *
+from tensorflow.python.ops.rnn_cell_impl import _Linear
+
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variable_scope as vs
+
+
+class VecAttGRUCell(RNNCell):
+    def __init__(self,
+                 num_units,
+                 activation=None,
+                 reuse=None,
+                 kernel_initializer=None,
+                 bias_initializer=None):
+        super(VecAttGRUCell, self).__init__(_reuse=reuse)
+        self._num_units = num_units
+        self._activation = activation or math_ops.tanh
+        self._kernel_initializer = kernel_initializer
+        self._bias_initializer = bias_initializer
+        self._gate_linear = None
+        self._candidate_linear = None
+
+    @property
+    def state_size(self):
+        return self._num_units
+
+    @property
+    def output_size(self):
+        return self._num_units
+
+    # 一个类实例也可以变成一个可调用对象，只需要实现一个特殊方法__call__()。
+    def __call__(self, inputs, state, att_score):
+        return self.call(inputs, state, att_score)
+
+    def call(self, inputs, state, att_score=None):
+        if self._gate_linear is None:
+            bias_ones = self._bias_initializer
+            if self._bias_initializer is None:
+                bias_ones = init_ops.constant_initializer(1.0, dtype=inputs.dtype)
+            with vs.variable_scope("gates"):
+                self._gate_linear = _Linear(
+                    [inputs, state],
+                    2 * self._num_units,
+                    True,
+                    bias_initializer=bias_ones,
+                    kernel_initializer=self._kernel_initializer
+                )
+
+        value = math_ops.sigmoid(self._gate_linear([inputs, state]))
+        r, u = array_ops.split(value, num_or_size_splits=2, axis=1)
+        r_state = r * state
+        if self._candidate_linear is None:
+            with vs.variable_scope("candidate"):
+                self._candidate_linear = _Linear(
+                    [inputs, r_state],
+                    self._num_units,
+                    True,
+                    bias_initializer=self._bias_initializer,
+                    kernel_initializer=self._kernel_initializer
+                )
+
+        c = self._activation(self._candidate_linear([inputs, r_state]))
+        u = (1.0 - att_score) * u
+        new_h = u * state + (1 - u) * c
+        return new_h, new_h
diff --git a/recommendation/Basic-DIN-Demo/model.py b/recommendation/Basic-DIN-Demo/model.py
index 24703045..556d3e70 100644
--- a/recommendation/Basic-DIN-Demo/model.py
+++ b/recommendation/Basic-DIN-Demo/model.py
@@ -129,11 +129,6 @@ def __init__(self,user_count,item_count,cate_count,cate_list):
         trainable_params = tf.trainable_variables()
         self.train_op = tf.train.GradientDescentOptimizer(learning_rate=self.lr).minimize(self.loss)
 
-
-
-
-
-
     def train(self,sess,uij,l):
         loss,_ = sess.run([self.loss,self.train_op],feed_dict={
             #self.u : uij[0],
diff --git a/recommendation/recommendation-FM-demo/FM_model.py b/recommendation/recommendation-FM-demo/FM_model.py
index 915a07cb..8063a9ed 100644
--- a/recommendation/recommendation-FM-demo/FM_model.py
+++ b/recommendation/recommendation-FM-demo/FM_model.py
@@ -13,7 +13,7 @@ def vectorize_dic(dic,ix=None,p=None,n=0,g=0):
     """
     dic -- dictionary of feature lists. Keys are the name of features
     ix -- index generator (default None)
-    p -- dimension of featrure space (number of columns in the sparse matrix) (default None)
+    p -- dimension of feature space (number of columns in the sparse matrix) (default None)
     """
     if ix==None:
         ix = dict()