Fix lightning tests for 0.36 feature changes (PennyLaneAI#5450)

### Before submitting

Please complete the following checklist when submitting a PR:

- [x] All new features must include a unit test.
If you've fixed a bug or added code that should be tested, add a test to
the
      test directory!

- [x] All new functions and code must be clearly commented and
documented.
If you do make documentation changes, make sure that the docs build and
      render correctly by running `make docs`.

- [x] Ensure that the test suite passes, by running `make test`.

- [x] Add a new entry to the `doc/releases/changelog-dev.md` file,
summarizing the
      change, and including a link back to the PR.

- [x] The PennyLane source code conforms to
      [PEP8 standards](https://www.python.org/dev/peps/pep-0008/).
We check all of our code against [Pylint](https://www.pylint.org/).
      To lint modified files, simply `pip install pylint`, and then
      run `pylint pennylane/path/to/file.py`.

When all the above are checked, delete everything above the dashed
line and fill in the pull request template.


------------------------------------------------------------------------------------------------------------

**Context:** This fixes the current xfail tests against Lightning, of
which was resolved with the latest merges into the Lightning repo.

**Description of the Change:**

**Benefits:**

**Possible Drawbacks:**

**Related GitHub Issues:**

doc/releases/changelog-dev.md

@@ -291,6 +291,9 @@
 
 <h3>Bug fixes 🐛</h3>
 
+* Fix failing tests due to changes with Lightning's adjoint diff pipeline.
+  [(#5450)](https://github.com/PennyLaneAI/pennylane/pull/5450)
+
 * Fix Torch tensor locality with autoray-registered coerce method.
   [(#5438)](https://github.com/PennyLaneAI/pennylane/pull/5438)
 

tests/optimize/test_spsa.py

@@ -479,7 +479,6 @@ def cost_fun(params, num_qubits=1):
         assert np.all(params != init_params)
         assert energy < init_energy
 
-    @pytest.mark.xfail(reason="Lightning cannot use adjoint with state prep ops yet")
     @pytest.mark.slow
     def test_lighting_device(self):
         """Test SPSAOptimizer implementation with lightning.qubit device."""

tests/transforms/test_defer_measurements.py

@@ -748,12 +748,7 @@ def test_quantum_teleportation(self, rads):
         [
             "default.qubit",
             "default.mixed",
-            pytest.param(
-                "lightning.qubit",
-                marks=pytest.mark.xfail(
-                    reason="Lightning adjoint decomposition needs to be updated"
-                ),
-            ),
+            "lightning.qubit",
         ],
     )
     @pytest.mark.parametrize("ops", [(qml.RX, qml.CRX), (qml.RY, qml.CRY), (qml.RZ, qml.CRZ)])
@@ -857,12 +852,7 @@ def test_hamiltonian_queued(self):
         [
             "default.qubit",
             "default.mixed",
-            pytest.param(
-                "lightning.qubit",
-                marks=pytest.mark.xfail(
-                    reason="Lightning adjoint decomposition needs to be updated"
-                ),
-            ),
+            "lightning.qubit",
         ],
     )
     @pytest.mark.parametrize("ops", [(qml.RX, qml.CRX), (qml.RY, qml.CRY), (qml.RZ, qml.CRZ)])
@@ -899,12 +889,7 @@ def quantum_control_circuit(rads):
         [
             "default.qubit",
             "default.mixed",
-            pytest.param(
-                "lightning.qubit",
-                marks=pytest.mark.xfail(
-                    reason="Lightning adjoint decomposition needs to be updated"
-                ),
-            ),
+            "lightning.qubit",
         ],
     )
     def test_conditional_rotations_with_else(self, device):
@@ -980,16 +965,7 @@ def qnode():
 
     @pytest.mark.parametrize(
         "device",
-        [
-            "default.qubit",
-            "default.mixed",
-            pytest.param(
-                "lightning.qubit",
-                marks=pytest.mark.xfail(
-                    reason="Lightning adjoint decomposition needs to be updated"
-                ),
-            ),
-        ],
+        ["default.qubit", "default.mixed", "lightning.qubit"],
     )
     def test_cond_qfunc(self, device):
         """Test that a qfunc can also used with qml.cond."""
@@ -1026,16 +1002,7 @@ def quantum_control_circuit(r):
 
     @pytest.mark.parametrize(
         "device",
-        [
-            "default.qubit",
-            "default.mixed",
-            pytest.param(
-                "lightning.qubit",
-                marks=pytest.mark.xfail(
-                    reason="Lightning adjoint decomposition needs to be updated"
-                ),
-            ),
-        ],
+        ["default.qubit", "default.mixed", "lightning.qubit"],
     )
     def test_cond_qfunc_with_else(self, device):
         """Test that a qfunc can also used with qml.cond even when an else