From 81bf7838a49d2db9d30e14f8584a2fafbd5e8f4d Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 10 Oct 2017 15:36:43 -0700 Subject: [PATCH 01/19] Add basic decorators to track model provenance. --- modules/atom/include/provenance.h | 100 +++++++++++++++++++++++++++ modules/atom/pyext/swig.i-in | 3 + modules/atom/src/provenance.cpp | 30 ++++++++ modules/atom/test/test_provenance.py | 31 +++++++++ 4 files changed, 164 insertions(+) create mode 100644 modules/atom/include/provenance.h create mode 100644 modules/atom/src/provenance.cpp create mode 100644 modules/atom/test/test_provenance.py diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h new file mode 100644 index 0000000000..3d0d3cc7c2 --- /dev/null +++ b/modules/atom/include/provenance.h @@ -0,0 +1,100 @@ +/** + * \file IMP/atom/provenance.h + * \brief Classes to track how the model was created. + * + * Copyright 2007-2017 IMP Inventors. All rights reserved. + */ + +#ifndef IMPATOM_PROVENANCE_H +#define IMPATOM_PROVENANCE_H + +#include + +#include +#include +#include +#include +#include + +IMPATOM_BEGIN_NAMESPACE + +//! Track how parts of the system were created. +/** Particles are linked with this decorator into a directed acyclic graph + that tracks all IMP transformations of the system all the way back to + raw inputs (such as PDB files). + + Typically, part of an IMP::Model (usually an atom::Hierarchy particle) + is decorated with Provenanced that points to the root of this graph. + */ +class IMPATOMEXPORT Provenance : public Decorator { + static void do_setup_particle(Model *m, ParticleIndex pi) { + // Use self-index to indicate no previous state is set yet + m->add_attribute(get_previous_state_key(), pi, pi); + } + +public: + static ParticleIndexKey get_previous_state_key(); + + static bool get_is_setup(Model *m, ParticleIndex pi) { + return m->get_has_attribute(get_previous_state_key(), pi); + } + + //! \return the previous state, or Provenance() if none exists. + Provenance get_previous_state() const { + ParticleIndex pi = get_model()->get_attribute(get_previous_state_key(), + get_particle_index()); + // self-index indicates no previous state is set yet + if (pi == get_particle_index()) { + return Provenance(); + } else { + return Provenance(get_model(), pi); + } + } + + //! Set the previous state + /** It is considered an error to try to set this more than once. */ + void set_previous_state(Provenance p) { + IMP_USAGE_CHECK(get_model()->get_attribute(get_previous_state_key(), + get_particle_index()) + == get_particle_index(), + "Previous state is already set"); + get_model()->set_attribute(get_previous_state_key(), + get_particle_index(), p.get_particle_index()); + } + + IMP_DECORATOR_METHODS(Provenance, Decorator); + IMP_DECORATOR_SETUP_0(Provenance); +}; + +//! Tag part of the system to track how it was created. +class IMPATOMEXPORT Provenanced : public Decorator { + static void do_setup_particle(Model *m, ParticleIndex pi, + Provenance p) { + m->add_attribute(get_provenance_key(), pi, p.get_particle_index()); + } + +public: + static ParticleIndexKey get_provenance_key(); + + static bool get_is_setup(Model *m, ParticleIndex pi) { + return m->get_has_attribute(get_provenance_key(), pi); + } + + Provenance get_provenance() const { + ParticleIndex pi = get_model()->get_attribute(get_provenance_key(), + get_particle_index()); + return Provenance(get_model(), pi); + } + + void set_provenance(Provenance p) const { + get_model()->set_attribute(get_provenance_key(), get_particle_index(), + p.get_particle_index()); + } + + IMP_DECORATOR_METHODS(Provenanced, Decorator); + IMP_DECORATOR_SETUP_1(Provenanced, Provenance, p); +}; + +IMPATOM_END_NAMESPACE + +#endif /* IMPATOM_PROVENANCE_H */ diff --git a/modules/atom/pyext/swig.i-in b/modules/atom/pyext/swig.i-in index 46a4b6f763..bf640acb36 100644 --- a/modules/atom/pyext/swig.i-in +++ b/modules/atom/pyext/swig.i-in @@ -37,6 +37,8 @@ IMP_SWIG_DECORATOR(IMP::atom, State, States); IMP_SWIG_DECORATOR(IMP::atom, Copy, Copies); IMP_SWIG_DECORATOR(IMP::atom, SecondaryStructureResidue, SecondaryStructureResidues); IMP_SWIG_DECORATOR(IMP::atom, StructureSource, StructureSources); +IMP_SWIG_DECORATOR(IMP::atom, Provenance, Provenances); +IMP_SWIG_DECORATOR(IMP::atom, Provenanced, Provenanceds); IMP_SWIG_BASE_OBJECT(IMP::atom, Simulator, Simulators); IMP_SWIG_DIRECTOR(IMP::atom, PDBSelector); IMP_SWIG_OBJECT(IMP::atom, ATOMPDBSelector, ATOMPDBSelectors); @@ -236,6 +238,7 @@ IMP_SWIG_VALUE_INSTANCE(IMP::atom, CHARMMAngle, CHARMMConnection, CHARMMAngles); %include "IMP/atom/Mass.h" %include "IMP/atom/BondedPairFilter.h" %include "IMP/atom/mol2.h" +%include "IMP/atom/provenance.h" %include "IMP/atom/SecondaryStructureResidue.h" %include "IMP/atom/secondary_structure_reader.h" diff --git a/modules/atom/src/provenance.cpp b/modules/atom/src/provenance.cpp new file mode 100644 index 0000000000..ddd005d6e8 --- /dev/null +++ b/modules/atom/src/provenance.cpp @@ -0,0 +1,30 @@ +/** + * \file provenance.cpp + * \brief Classes to track how the model was created. + * + * Copyright 2007-2017 IMP Inventors. All rights reserved. + */ + +#include + +IMPATOM_BEGIN_NAMESPACE + +ParticleIndexKey Provenanced::get_provenance_key() { + static const ParticleIndexKey provenance("provenance"); + return provenance; +} + +void Provenanced::show(std::ostream &out) const { + out << "Provenanced" << std::endl; +} + +ParticleIndexKey Provenance::get_previous_state_key() { + static const ParticleIndexKey previous_state("previous_state"); + return previous_state; +} + +void Provenance::show(std::ostream &out) const { + out << "Provenance" << std::endl; +} + +IMPATOM_END_NAMESPACE diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py new file mode 100644 index 0000000000..ad8b0ea0dc --- /dev/null +++ b/modules/atom/test/test_provenance.py @@ -0,0 +1,31 @@ +from __future__ import print_function +import IMP +import IMP.test +import IMP.atom + +class Tests(IMP.test.TestCase): + + def test_provenance(self): + """Test Provenance decorator""" + m = IMP.Model() + p = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) + self.assertTrue(IMP.atom.Provenance.get_is_setup(p)) + self.assertFalse(p.get_previous_state()) + + p2 = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) + p.set_previous_state(p2) + self.assertEqual(p.get_previous_state(), p2) + + def test_provenanced(self): + """Test Provenanced decorator""" + m = IMP.Model() + p = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) + pd = IMP.atom.Provenanced.setup_particle(m, IMP.Particle(m), p) + self.assertEqual(pd.get_provenance(), p) + pd.set_provenance(p) + self.assertTrue(IMP.atom.Provenanced.get_is_setup(pd)) + self.assertFalse(IMP.atom.Provenanced.get_is_setup(p)) + self.assertFalse(IMP.atom.Provenance.get_is_setup(pd)) + +if __name__ == '__main__': + IMP.test.main() From fc3c4277ec8e2a65a382626dad8aafefaca5718d Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 10 Oct 2017 16:09:46 -0700 Subject: [PATCH 02/19] Add a decorator to track structure provenance. --- modules/atom/include/provenance.h | 51 ++++++++++++++++++++++++++-- modules/atom/pyext/swig.i-in | 1 + modules/atom/src/provenance.cpp | 15 ++++++++ modules/atom/test/test_provenance.py | 9 +++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h index 3d0d3cc7c2..0fd64a646d 100644 --- a/modules/atom/include/provenance.h +++ b/modules/atom/include/provenance.h @@ -32,9 +32,9 @@ class IMPATOMEXPORT Provenance : public Decorator { m->add_attribute(get_previous_state_key(), pi, pi); } -public: static ParticleIndexKey get_previous_state_key(); +public: static bool get_is_setup(Model *m, ParticleIndex pi) { return m->get_has_attribute(get_previous_state_key(), pi); } @@ -66,6 +66,53 @@ class IMPATOMEXPORT Provenance : public Decorator { IMP_DECORATOR_SETUP_0(Provenance); }; +//! Track creation of a system fragment from a PDB file. +class IMPATOMEXPORT StructureProvenance : public Provenance { + static void do_setup_particle(Model *m, ParticleIndex pi, + std::string filename, + std::string chain_id) { + IMP_USAGE_CHECK(!filename.empty(), "The filename cannot be empty."); + m->add_attribute(get_filename_key(), pi, filename); + m->add_attribute(get_chain_key(), pi, chain_id); + } + + static StringKey get_filename_key(); + static StringKey get_chain_key(); + +public: + static bool get_is_setup(Model *m, ParticleIndex pi) { + return m->get_has_attribute(get_filename_key(), pi) + && m->get_has_attribute(get_chain_key(), pi); + } + + //! Set the filename + void set_filename(std::string filename) const { + IMP_USAGE_CHECK(!filename.empty(), "The filename cannot be empty"); + return get_model()->set_attribute(get_filename_key(), get_particle_index(), + filename); + } + + //! \return the filename + std::string get_filename() const { + return get_model()->get_attribute(get_filename_key(), get_particle_index()); + } + + //! Set the chain ID + void set_chain_id(std::string chain_id) const { + return get_model()->set_attribute(get_chain_key(), get_particle_index(), + chain_id); + } + + //! \return the chain ID + std::string get_chain_id() const { + return get_model()->get_attribute(get_chain_key(), get_particle_index()); + } + + IMP_DECORATOR_METHODS(StructureProvenance, Provenance); + IMP_DECORATOR_SETUP_2(StructureProvenance, std::string, filename, + std::string, chain_id); +}; + //! Tag part of the system to track how it was created. class IMPATOMEXPORT Provenanced : public Decorator { static void do_setup_particle(Model *m, ParticleIndex pi, @@ -73,8 +120,8 @@ class IMPATOMEXPORT Provenanced : public Decorator { m->add_attribute(get_provenance_key(), pi, p.get_particle_index()); } -public: static ParticleIndexKey get_provenance_key(); +public: static bool get_is_setup(Model *m, ParticleIndex pi) { return m->get_has_attribute(get_provenance_key(), pi); diff --git a/modules/atom/pyext/swig.i-in b/modules/atom/pyext/swig.i-in index bf640acb36..ed0d2d9dee 100644 --- a/modules/atom/pyext/swig.i-in +++ b/modules/atom/pyext/swig.i-in @@ -38,6 +38,7 @@ IMP_SWIG_DECORATOR(IMP::atom, Copy, Copies); IMP_SWIG_DECORATOR(IMP::atom, SecondaryStructureResidue, SecondaryStructureResidues); IMP_SWIG_DECORATOR(IMP::atom, StructureSource, StructureSources); IMP_SWIG_DECORATOR(IMP::atom, Provenance, Provenances); +IMP_SWIG_DECORATOR(IMP::atom, StructureProvenance, StructureProvenances); IMP_SWIG_DECORATOR(IMP::atom, Provenanced, Provenanceds); IMP_SWIG_BASE_OBJECT(IMP::atom, Simulator, Simulators); IMP_SWIG_DIRECTOR(IMP::atom, PDBSelector); diff --git a/modules/atom/src/provenance.cpp b/modules/atom/src/provenance.cpp index ddd005d6e8..2435abf2ef 100644 --- a/modules/atom/src/provenance.cpp +++ b/modules/atom/src/provenance.cpp @@ -27,4 +27,19 @@ void Provenance::show(std::ostream &out) const { out << "Provenance" << std::endl; } +StringKey StructureProvenance::get_filename_key() { + static const StringKey filename("sp_filename"); + return filename; +} + +StringKey StructureProvenance::get_chain_key() { + static const StringKey chain("sp_chain"); + return chain; +} + +void StructureProvenance::show(std::ostream &out) const { + out << "StructureProvenance " << get_filename() << " " << get_chain_id() + << std::endl; +} + IMPATOM_END_NAMESPACE diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py index ad8b0ea0dc..67323f718a 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/atom/test/test_provenance.py @@ -16,6 +16,15 @@ def test_provenance(self): p.set_previous_state(p2) self.assertEqual(p.get_previous_state(), p2) + def test_structure_provenance(self): + """Test StructureProvenance decorator""" + m = IMP.Model() + p = IMP.atom.StructureProvenance.setup_particle(m, IMP.Particle(m), + "testfile", "testchain") + self.assertTrue(IMP.atom.StructureProvenance.get_is_setup(p)) + self.assertEqual(p.get_filename(), "testfile") + self.assertEqual(p.get_chain_id(), "testchain") + def test_provenanced(self): """Test Provenanced decorator""" m = IMP.Model() From 32624d508226d17a59c74cf28683b12d74473119 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 10 Oct 2017 17:33:34 -0700 Subject: [PATCH 03/19] Add function to set provenance for a particle. --- modules/atom/include/provenance.h | 4 ++++ modules/atom/src/provenance.cpp | 12 ++++++++++++ modules/atom/test/test_provenance.py | 19 +++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h index 0fd64a646d..4f593eb7e1 100644 --- a/modules/atom/include/provenance.h +++ b/modules/atom/include/provenance.h @@ -142,6 +142,10 @@ class IMPATOMEXPORT Provenanced : public Decorator { IMP_DECORATOR_SETUP_1(Provenanced, Provenance, p); }; +//! Add provenance to part of the model. +IMPATOMEXPORT void add_provenance(Model *m, ParticleIndex pi, + Provenance p); + IMPATOM_END_NAMESPACE #endif /* IMPATOM_PROVENANCE_H */ diff --git a/modules/atom/src/provenance.cpp b/modules/atom/src/provenance.cpp index 2435abf2ef..a015a092c5 100644 --- a/modules/atom/src/provenance.cpp +++ b/modules/atom/src/provenance.cpp @@ -42,4 +42,16 @@ void StructureProvenance::show(std::ostream &out) const { << std::endl; } +void add_provenance(Model *m, ParticleIndex pi, Provenance p) { + if (Provenanced::get_is_setup(m, pi)) { + // add the new provenance as a new root + Provenanced pd = Provenanced(m, pi); + Provenance old_provenance = pd.get_provenance(); + p.set_previous_state(old_provenance); + pd.set_provenance(p); + } else { + Provenanced::setup_particle(m, pi, p); + } +} + IMPATOM_END_NAMESPACE diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py index 67323f718a..65ef918af8 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/atom/test/test_provenance.py @@ -36,5 +36,24 @@ def test_provenanced(self): self.assertFalse(IMP.atom.Provenanced.get_is_setup(p)) self.assertFalse(IMP.atom.Provenance.get_is_setup(pd)) + def test_add_provenance(self): + """Test add_provenance()""" + m = IMP.Model() + prov1 = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) + prov2 = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) + + p = IMP.Particle(m) + self.assertFalse(IMP.atom.Provenanced.get_is_setup(p)) + + IMP.atom.add_provenance(m, p, prov1) + self.assertTrue(IMP.atom.Provenanced.get_is_setup(p)) + pd = IMP.atom.Provenanced(p) + self.assertTrue(pd.get_provenance(), prov1) + + IMP.atom.add_provenance(m, p, prov2) + self.assertTrue(IMP.atom.Provenanced.get_is_setup(p)) + self.assertTrue(pd.get_provenance(), prov2) + self.assertTrue(pd.get_provenance().get_previous_state(), prov1) + if __name__ == '__main__': IMP.test.main() From b64b536750d64d7fc08c6d3785cac36700702d7f Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 10 Oct 2017 17:34:14 -0700 Subject: [PATCH 04/19] Set provenance when reading in PDB files. For each chain read from a PDB file, set a StructureProvenance so that we know the filename and chain ID the structure was ultimately derived from. --- modules/atom/src/pdb.cpp | 12 ++++++++++-- modules/atom/test/test_pdb.py | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/atom/src/pdb.cpp b/modules/atom/src/pdb.cpp index 7ffa78fcf0..345429be95 100644 --- a/modules/atom/src/pdb.cpp +++ b/modules/atom/src/pdb.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -244,11 +245,18 @@ Particle* residue_particle(Model* m, const std::string& pdb_line) { return p; } -Particle* chain_particle(Model* m, char chain_id) { +Particle* chain_particle(Model* m, char chain_id, std::string filename) { Particle* p = new Particle(m); Chain::setup_particle(p, chain_id); p->set_name(std::string("Chain " + std::string(1, chain_id))); Molecule::setup_particle(p); + + // Set provenance of this chain + StructureProvenance sp + = StructureProvenance::setup_particle(new Particle(m), + filename, std::string(1, chain_id)); + add_provenance(m, p->get_index(), sp); + return p; } } @@ -350,7 +358,7 @@ Hierarchies read_pdb(std::istream& in, std::string name, Model* model, if (cp == nullptr || chain != curr_chain) { curr_chain = chain; // create new chain particle - cp = chain_particle(model, chain); + cp = chain_particle(model, chain, name); chain_name_set = false; Hierarchy(root_p).add_child(Chain(cp)); rp = nullptr; // make sure we get a new residue diff --git a/modules/atom/test/test_pdb.py b/modules/atom/test/test_pdb.py index dae6551b4d..3228098b45 100644 --- a/modules/atom/test/test_pdb.py +++ b/modules/atom/test/test_pdb.py @@ -46,7 +46,7 @@ def test_read(self): #! read PDB mp = IMP.atom.read_pdb(self.open_input_file("input.pdb"), m, IMP.atom.NonWaterPDBSelector()) - self.assertEqual(len(m.get_particle_indexes()), 1132) + self.assertEqual(len(m.get_particle_indexes()), 1133) # IMP.atom.show_molecular_hierarchy(mp) IMP.atom.show(mp) IMP.atom.add_bonds(mp) @@ -58,7 +58,7 @@ def test_read(self): m2 = IMP.Model() mp = IMP.atom.read_pdb(self.open_input_file("input.pdb"), m2, IMP.atom.CAlphaPDBSelector()) - self.assertEqual(len(m2.get_particle_indexes()), 260) + self.assertEqual(len(m2.get_particle_indexes()), 261) ps = IMP.atom.get_by_type(mp, IMP.atom.ATOM_TYPE) self.assertEqual(len(ps), 129) IMP.atom.add_bonds(mp) @@ -218,5 +218,21 @@ def test_indexes(self): lvs = IMP.atom.get_leaves(hp) self.assertEqual(IMP.atom.Atom(lvs[2]).get_input_index(), 3) + def test_provenance(self): + """Test that StructureProvenance is set""" + m = IMP.Model() + fname = self.get_input_file_name("hydrogen.pdb") + mp = IMP.atom.read_pdb(fname, m) + chains = IMP.atom.get_by_type(mp, IMP.atom.CHAIN_TYPE) + self.assertEqual(len(chains), 5) + for c in chains: + self.assertTrue(IMP.atom.Provenanced.get_is_setup(c)) + p = IMP.atom.Provenanced(c).get_provenance() + self.assertTrue(IMP.atom.StructureProvenance.get_is_setup(p)) + sp = IMP.atom.StructureProvenance(p) + self.assertEqual(sp.get_filename(), fname) + self.assertEqual(sp.get_chain_id(), IMP.atom.Chain(c).get_id()) + + if __name__ == '__main__': IMP.test.main() From f330be2801ceeacf4f2de84568fe59348a889c8b Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Mon, 16 Oct 2017 12:16:23 -0700 Subject: [PATCH 05/19] Add decorator to track sampling provenance. The SampleProvenance decorator can be used to tag part of the system (usually the atom::Hierarchy root) as being one of a number of frames generated by some sampling method, such as Monte Carlo. --- modules/atom/include/provenance.h | 71 ++++++++++++++++++++++++++++ modules/atom/pyext/swig.i-in | 1 + modules/atom/src/provenance.cpp | 29 ++++++++++++ modules/atom/test/test_provenance.py | 18 +++++++ 4 files changed, 119 insertions(+) diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h index 4f593eb7e1..8136043a50 100644 --- a/modules/atom/include/provenance.h +++ b/modules/atom/include/provenance.h @@ -15,6 +15,7 @@ #include #include #include +#include IMPATOM_BEGIN_NAMESPACE @@ -113,6 +114,76 @@ class IMPATOMEXPORT StructureProvenance : public Provenance { std::string, chain_id); }; +//! Track creation of a system fragment from sampling. +/** Part of the system (usually the top of a Hierarchy) tagged with this + decorator is understood to be a single frame from an ensemble of + multiple frames generated with some sampling method (e.g. Monte Carlo). + Additionally, the number of iterations of the sampler used to generate + each frame can be stored, if known and applicable. + The rest of the frames are generally stored in a file (e.g. an RMF file). + */ +class IMPATOMEXPORT SampleProvenance : public Provenance { + static void do_setup_particle(Model *m, ParticleIndex pi, + std::string method, int frames) { + m->add_attribute(get_method_key(), pi, method); + m->add_attribute(get_frames_key(), pi, frames); + m->add_attribute(get_iterations_key(), pi, 1); + } + + static StringKey get_method_key(); + static IntKey get_frames_key(); + static IntKey get_iterations_key(); + + static std::set& get_allowed_methods(); + +public: + static bool get_is_setup(Model *m, ParticleIndex pi) { + return m->get_has_attribute(get_method_key(), pi) + && m->get_has_attribute(get_iterations_key(), pi) + && m->get_has_attribute(get_frames_key(), pi); + } + + //! Set the sampling method + void set_method(std::string method) const { + IMP_USAGE_CHECK(get_allowed_methods().find(method) + != get_allowed_methods().end(), + "Invalid sampling method"); + return get_model()->set_attribute(get_method_key(), get_particle_index(), + method); + } + + //! \return the sampling method + std::string get_method() const { + return get_model()->get_attribute(get_method_key(), get_particle_index()); + } + + //! Set the number of frames + void set_number_of_frames(int frames) const { + return get_model()->set_attribute(get_frames_key(), get_particle_index(), + frames); + } + + //! \return the number of frames + int get_number_of_frames() const { + return get_model()->get_attribute(get_frames_key(), get_particle_index()); + } + + //! Set the number of iterations + void set_number_of_iterations(int iterations) const { + return get_model()->set_attribute(get_iterations_key(), + get_particle_index(), iterations); + } + + //! \return the number of iterations + int get_number_of_iterations() const { + return get_model()->get_attribute(get_iterations_key(), + get_particle_index()); + } + + IMP_DECORATOR_METHODS(SampleProvenance, Provenance); + IMP_DECORATOR_SETUP_2(SampleProvenance, std::string, method, int, frames); +}; + //! Tag part of the system to track how it was created. class IMPATOMEXPORT Provenanced : public Decorator { static void do_setup_particle(Model *m, ParticleIndex pi, diff --git a/modules/atom/pyext/swig.i-in b/modules/atom/pyext/swig.i-in index ed0d2d9dee..7dbe9b43bb 100644 --- a/modules/atom/pyext/swig.i-in +++ b/modules/atom/pyext/swig.i-in @@ -39,6 +39,7 @@ IMP_SWIG_DECORATOR(IMP::atom, SecondaryStructureResidue, SecondaryStructureResid IMP_SWIG_DECORATOR(IMP::atom, StructureSource, StructureSources); IMP_SWIG_DECORATOR(IMP::atom, Provenance, Provenances); IMP_SWIG_DECORATOR(IMP::atom, StructureProvenance, StructureProvenances); +IMP_SWIG_DECORATOR(IMP::atom, SampleProvenance, SampleProvenances); IMP_SWIG_DECORATOR(IMP::atom, Provenanced, Provenanceds); IMP_SWIG_BASE_OBJECT(IMP::atom, Simulator, Simulators); IMP_SWIG_DIRECTOR(IMP::atom, PDBSelector); diff --git a/modules/atom/src/provenance.cpp b/modules/atom/src/provenance.cpp index a015a092c5..a55c7f4249 100644 --- a/modules/atom/src/provenance.cpp +++ b/modules/atom/src/provenance.cpp @@ -42,6 +42,35 @@ void StructureProvenance::show(std::ostream &out) const { << std::endl; } +std::set& SampleProvenance::get_allowed_methods() { + static std::set m; + if (m.empty()) { + m.insert("Monte Carlo"); + m.insert("Molecular Dynamics"); + } + return m; +} + +StringKey SampleProvenance::get_method_key() { + static const StringKey method("sp_method"); + return method; +} + +IntKey SampleProvenance::get_frames_key() { + static const IntKey frames("sp_frames"); + return frames; +} + +IntKey SampleProvenance::get_iterations_key() { + static const IntKey iterations("sp_iterations"); + return iterations; +} + +void SampleProvenance::show(std::ostream &out) const { + out << "SampleProvenance " << get_number_of_frames() << " of " + << get_method() << std::endl; +} + void add_provenance(Model *m, ParticleIndex pi, Provenance p) { if (Provenanced::get_is_setup(m, pi)) { // add the new provenance as a new root diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py index 65ef918af8..6926e8f1aa 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/atom/test/test_provenance.py @@ -25,6 +25,24 @@ def test_structure_provenance(self): self.assertEqual(p.get_filename(), "testfile") self.assertEqual(p.get_chain_id(), "testchain") + def test_sample_provenance(self): + """Test SampleProvenance decorator""" + m = IMP.Model() + p = IMP.atom.SampleProvenance.setup_particle(m, IMP.Particle(m), + "Monte Carlo", 100) + self.assertTrue(IMP.atom.SampleProvenance.get_is_setup(p)) + self.assertEqual(p.get_method(), "Monte Carlo") + p.set_method("Molecular Dynamics") + self.assertEqual(p.get_method(), "Molecular Dynamics") + if IMP.get_check_level() == IMP.USAGE_AND_INTERNAL: + self.assertRaises(IMP.UsageError, p.set_method, "Garbage") + self.assertEqual(p.get_number_of_frames(), 100) + p.set_number_of_frames(200) + self.assertEqual(p.get_number_of_frames(), 200) + self.assertEqual(p.get_number_of_iterations(), 1) + p.set_number_of_iterations(42) + self.assertEqual(p.get_number_of_iterations(), 42) + def test_provenanced(self): """Test Provenanced decorator""" m = IMP.Model() From b9592d1c80120c0521215b10c03bf7543815b140 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Mon, 16 Oct 2017 12:37:04 -0700 Subject: [PATCH 06/19] Add decorator to track creation of models by clustering. --- modules/atom/include/provenance.h | 33 ++++++++++++++++++++++++++++ modules/atom/pyext/swig.i-in | 1 + modules/atom/src/provenance.cpp | 10 +++++++++ modules/atom/test/test_provenance.py | 9 ++++++++ 4 files changed, 53 insertions(+) diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h index 8136043a50..c0629d932d 100644 --- a/modules/atom/include/provenance.h +++ b/modules/atom/include/provenance.h @@ -184,6 +184,39 @@ class IMPATOMEXPORT SampleProvenance : public Provenance { IMP_DECORATOR_SETUP_2(SampleProvenance, std::string, method, int, frames); }; +//! Track creation of a system fragment from clustering. +/** Part of the system (usually the top of a Hierarchy) tagged with this + decorator is understood to be a single frame inside a cluster of + specified size. The rest of the cluster members are generally stored + in a file (e.g. an RMF file). + */ +class IMPATOMEXPORT ClusterProvenance : public Provenance { + static void do_setup_particle(Model *m, ParticleIndex pi, int members) { + m->add_attribute(get_members_key(), pi, members); + } + + static IntKey get_members_key(); + +public: + static bool get_is_setup(Model *m, ParticleIndex pi) { + return m->get_has_attribute(get_members_key(), pi); + } + + //! Set the number of cluster members + void set_number_of_members(int members) const { + return get_model()->set_attribute(get_members_key(), get_particle_index(), + members); + } + + //! \return the number of cluster members + int get_number_of_members() const { + return get_model()->get_attribute(get_members_key(), get_particle_index()); + } + + IMP_DECORATOR_METHODS(ClusterProvenance, Provenance); + IMP_DECORATOR_SETUP_1(ClusterProvenance, int, members); +}; + //! Tag part of the system to track how it was created. class IMPATOMEXPORT Provenanced : public Decorator { static void do_setup_particle(Model *m, ParticleIndex pi, diff --git a/modules/atom/pyext/swig.i-in b/modules/atom/pyext/swig.i-in index 7dbe9b43bb..e3f9538e7c 100644 --- a/modules/atom/pyext/swig.i-in +++ b/modules/atom/pyext/swig.i-in @@ -40,6 +40,7 @@ IMP_SWIG_DECORATOR(IMP::atom, StructureSource, StructureSources); IMP_SWIG_DECORATOR(IMP::atom, Provenance, Provenances); IMP_SWIG_DECORATOR(IMP::atom, StructureProvenance, StructureProvenances); IMP_SWIG_DECORATOR(IMP::atom, SampleProvenance, SampleProvenances); +IMP_SWIG_DECORATOR(IMP::atom, ClusterProvenance, ClusterProvenances); IMP_SWIG_DECORATOR(IMP::atom, Provenanced, Provenanceds); IMP_SWIG_BASE_OBJECT(IMP::atom, Simulator, Simulators); IMP_SWIG_DIRECTOR(IMP::atom, PDBSelector); diff --git a/modules/atom/src/provenance.cpp b/modules/atom/src/provenance.cpp index a55c7f4249..99899d0f7c 100644 --- a/modules/atom/src/provenance.cpp +++ b/modules/atom/src/provenance.cpp @@ -71,6 +71,16 @@ void SampleProvenance::show(std::ostream &out) const { << get_method() << std::endl; } +IntKey ClusterProvenance::get_members_key() { + static const IntKey members("cp_members"); + return members; +} + +void ClusterProvenance::show(std::ostream &out) const { + out << "ClusterProvenance with " << get_number_of_members() + << " members" << std::endl; +} + void add_provenance(Model *m, ParticleIndex pi, Provenance p) { if (Provenanced::get_is_setup(m, pi)) { // add the new provenance as a new root diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py index 6926e8f1aa..d5d235e378 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/atom/test/test_provenance.py @@ -43,6 +43,15 @@ def test_sample_provenance(self): p.set_number_of_iterations(42) self.assertEqual(p.get_number_of_iterations(), 42) + def test_cluster_provenance(self): + """Test ClusterProvenance decorator""" + m = IMP.Model() + p = IMP.atom.ClusterProvenance.setup_particle(m, IMP.Particle(m), 10) + self.assertTrue(IMP.atom.ClusterProvenance.get_is_setup(p)) + self.assertEqual(p.get_number_of_members(), 10) + p.set_number_of_members(42) + self.assertEqual(p.get_number_of_members(), 42) + def test_provenanced(self): """Test Provenanced decorator""" m = IMP.Model() From 8f49993a375e2b4ae6f6791f63369a44c5737272 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Mon, 16 Oct 2017 13:09:44 -0700 Subject: [PATCH 07/19] Add decorator to track creation of models by combination. --- modules/atom/include/provenance.h | 50 ++++++++++++++++++++++++++++ modules/atom/pyext/swig.i-in | 1 + modules/atom/src/provenance.cpp | 16 +++++++++ modules/atom/test/test_provenance.py | 12 +++++++ 4 files changed, 79 insertions(+) diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h index c0629d932d..1e4b13cbac 100644 --- a/modules/atom/include/provenance.h +++ b/modules/atom/include/provenance.h @@ -184,6 +184,56 @@ class IMPATOMEXPORT SampleProvenance : public Provenance { IMP_DECORATOR_SETUP_2(SampleProvenance, std::string, method, int, frames); }; +//! Track creation of a system fragment by combination. +/** Part of the system (usually the top of a Hierarchy) tagged with this + decorator is understood to be a single frame within an ensemble that + was created by combining a number of independent runs. One of those runs + should be the previous state of this provenance. The runs should be + essentially identical, differing at most only in the number of frames. + The total size of the resulting ensemble is stored here. + */ +class IMPATOMEXPORT CombineProvenance : public Provenance { + static void do_setup_particle(Model *m, ParticleIndex pi, int runs, + int frames) { + m->add_attribute(get_runs_key(), pi, runs); + m->add_attribute(get_frames_key(), pi, frames); + } + + static IntKey get_runs_key(); + static IntKey get_frames_key(); + +public: + static bool get_is_setup(Model *m, ParticleIndex pi) { + return m->get_has_attribute(get_frames_key(), pi) + && m->get_has_attribute(get_runs_key(), pi); + } + + //! Set the total number of frames + void set_number_of_frames(int frames) const { + return get_model()->set_attribute(get_frames_key(), get_particle_index(), + frames); + } + + //! \return the total number of frames + int get_number_of_frames() const { + return get_model()->get_attribute(get_frames_key(), get_particle_index()); + } + + //! Set the number of runs + void set_number_of_runs(int runs) const { + return get_model()->set_attribute(get_runs_key(), get_particle_index(), + runs); + } + + //! \return the number of runs + int get_number_of_runs() const { + return get_model()->get_attribute(get_runs_key(), get_particle_index()); + } + + IMP_DECORATOR_METHODS(CombineProvenance, Provenance); + IMP_DECORATOR_SETUP_2(CombineProvenance, int, runs, int, frames); +}; + //! Track creation of a system fragment from clustering. /** Part of the system (usually the top of a Hierarchy) tagged with this decorator is understood to be a single frame inside a cluster of diff --git a/modules/atom/pyext/swig.i-in b/modules/atom/pyext/swig.i-in index e3f9538e7c..8c0769253d 100644 --- a/modules/atom/pyext/swig.i-in +++ b/modules/atom/pyext/swig.i-in @@ -41,6 +41,7 @@ IMP_SWIG_DECORATOR(IMP::atom, Provenance, Provenances); IMP_SWIG_DECORATOR(IMP::atom, StructureProvenance, StructureProvenances); IMP_SWIG_DECORATOR(IMP::atom, SampleProvenance, SampleProvenances); IMP_SWIG_DECORATOR(IMP::atom, ClusterProvenance, ClusterProvenances); +IMP_SWIG_DECORATOR(IMP::atom, CombineProvenance, CombineProvenances); IMP_SWIG_DECORATOR(IMP::atom, Provenanced, Provenanceds); IMP_SWIG_BASE_OBJECT(IMP::atom, Simulator, Simulators); IMP_SWIG_DIRECTOR(IMP::atom, PDBSelector); diff --git a/modules/atom/src/provenance.cpp b/modules/atom/src/provenance.cpp index 99899d0f7c..fd43f768a8 100644 --- a/modules/atom/src/provenance.cpp +++ b/modules/atom/src/provenance.cpp @@ -71,6 +71,22 @@ void SampleProvenance::show(std::ostream &out) const { << get_method() << std::endl; } +IntKey CombineProvenance::get_runs_key() { + static const IntKey runs("cp_runs"); + return runs; +} + +IntKey CombineProvenance::get_frames_key() { + static const IntKey frames("cp_frames"); + return frames; +} + +void CombineProvenance::show(std::ostream &out) const { + out << "CombineProvenance of " << get_number_of_runs() + << " runs resulting in " << get_number_of_frames() + << " frames" << std::endl; +} + IntKey ClusterProvenance::get_members_key() { static const IntKey members("cp_members"); return members; diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py index d5d235e378..a1d4a5438f 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/atom/test/test_provenance.py @@ -43,6 +43,18 @@ def test_sample_provenance(self): p.set_number_of_iterations(42) self.assertEqual(p.get_number_of_iterations(), 42) + def test_combine_provenance(self): + """Test CombineProvenance decorator""" + m = IMP.Model() + p = IMP.atom.CombineProvenance.setup_particle(m, IMP.Particle(m), 5, 42) + self.assertTrue(IMP.atom.CombineProvenance.get_is_setup(p)) + self.assertEqual(p.get_number_of_runs(), 5) + p.set_number_of_runs(7) + self.assertEqual(p.get_number_of_runs(), 7) + self.assertEqual(p.get_number_of_frames(), 42) + p.set_number_of_frames(100) + self.assertEqual(p.get_number_of_frames(), 100) + def test_cluster_provenance(self): """Test ClusterProvenance decorator""" m = IMP.Model() From b163c93a309bc03298ae1281b9bd7033b9b9959e Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Mon, 16 Oct 2017 13:40:28 -0700 Subject: [PATCH 08/19] Add decorator to track creation of models by filtering. --- modules/atom/include/provenance.h | 49 ++++++++++++++++++++++++++++ modules/atom/pyext/swig.i-in | 1 + modules/atom/src/provenance.cpp | 15 +++++++++ modules/atom/test/test_provenance.py | 13 ++++++++ 4 files changed, 78 insertions(+) diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h index 1e4b13cbac..947eb5dce0 100644 --- a/modules/atom/include/provenance.h +++ b/modules/atom/include/provenance.h @@ -234,6 +234,55 @@ class IMPATOMEXPORT CombineProvenance : public Provenance { IMP_DECORATOR_SETUP_2(CombineProvenance, int, runs, int, frames); }; +//! Track creation of a system fragment by filtering. +/** Part of the system (usually the top of a Hierarchy) tagged with this + decorator is understood to be a single frame within an ensemble that + resulted from filtering a larger ensemble (the previous state of this + provenance) by discarding models with scores above the threshold. + */ +class IMPATOMEXPORT FilterProvenance : public Provenance { + static void do_setup_particle(Model *m, ParticleIndex pi, double threshold, + int frames) { + m->add_attribute(get_threshold_key(), pi, threshold); + m->add_attribute(get_frames_key(), pi, frames); + } + + static FloatKey get_threshold_key(); + static IntKey get_frames_key(); + +public: + static bool get_is_setup(Model *m, ParticleIndex pi) { + return m->get_has_attribute(get_threshold_key(), pi) + && m->get_has_attribute(get_frames_key(), pi); + } + + //! Set the number of frames + void set_number_of_frames(int frames) const { + return get_model()->set_attribute(get_frames_key(), get_particle_index(), + frames); + } + + //! \return the number of frames + int get_number_of_frames() const { + return get_model()->get_attribute(get_frames_key(), get_particle_index()); + } + + //! Set the score threshold + void set_threshold(double threshold) const { + return get_model()->set_attribute(get_threshold_key(), get_particle_index(), + threshold); + } + + //! \return the threshold + double get_threshold() const { + return get_model()->get_attribute(get_threshold_key(), + get_particle_index()); + } + + IMP_DECORATOR_METHODS(FilterProvenance, Provenance); + IMP_DECORATOR_SETUP_2(FilterProvenance, double, threshold, int, frames); +}; + //! Track creation of a system fragment from clustering. /** Part of the system (usually the top of a Hierarchy) tagged with this decorator is understood to be a single frame inside a cluster of diff --git a/modules/atom/pyext/swig.i-in b/modules/atom/pyext/swig.i-in index 8c0769253d..a3e2d06cd5 100644 --- a/modules/atom/pyext/swig.i-in +++ b/modules/atom/pyext/swig.i-in @@ -42,6 +42,7 @@ IMP_SWIG_DECORATOR(IMP::atom, StructureProvenance, StructureProvenances); IMP_SWIG_DECORATOR(IMP::atom, SampleProvenance, SampleProvenances); IMP_SWIG_DECORATOR(IMP::atom, ClusterProvenance, ClusterProvenances); IMP_SWIG_DECORATOR(IMP::atom, CombineProvenance, CombineProvenances); +IMP_SWIG_DECORATOR(IMP::atom, FilterProvenance, FilterProvenances); IMP_SWIG_DECORATOR(IMP::atom, Provenanced, Provenanceds); IMP_SWIG_BASE_OBJECT(IMP::atom, Simulator, Simulators); IMP_SWIG_DIRECTOR(IMP::atom, PDBSelector); diff --git a/modules/atom/src/provenance.cpp b/modules/atom/src/provenance.cpp index fd43f768a8..407d6ca18b 100644 --- a/modules/atom/src/provenance.cpp +++ b/modules/atom/src/provenance.cpp @@ -97,6 +97,21 @@ void ClusterProvenance::show(std::ostream &out) const { << " members" << std::endl; } +FloatKey FilterProvenance::get_threshold_key() { + static const FloatKey threshold("fp_threshold"); + return threshold; +} + +IntKey FilterProvenance::get_frames_key() { + static const IntKey frames("fp_frames"); + return frames; +} + +void FilterProvenance::show(std::ostream &out) const { + out << "FilterProvenance threshold " << get_threshold() + << " resulting in " << get_number_of_frames() << " frames" << std::endl; +} + void add_provenance(Model *m, ParticleIndex pi, Provenance p) { if (Provenanced::get_is_setup(m, pi)) { // add the new provenance as a new root diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py index a1d4a5438f..3b06f70f20 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/atom/test/test_provenance.py @@ -55,6 +55,19 @@ def test_combine_provenance(self): p.set_number_of_frames(100) self.assertEqual(p.get_number_of_frames(), 100) + def test_filter_provenance(self): + """Test FilterProvenance decorator""" + m = IMP.Model() + p = IMP.atom.FilterProvenance.setup_particle(m, IMP.Particle(m), 100.5, + 42) + self.assertTrue(IMP.atom.FilterProvenance.get_is_setup(p)) + self.assertAlmostEqual(p.get_threshold(), 100.5, delta=0.01) + p.set_threshold(76.0) + self.assertAlmostEqual(p.get_threshold(), 76.0, delta=0.01) + self.assertEqual(p.get_number_of_frames(), 42) + p.set_number_of_frames(100) + self.assertEqual(p.get_number_of_frames(), 100) + def test_cluster_provenance(self): """Test ClusterProvenance decorator""" m = IMP.Model() From 2a57a1d0d9aabe95712fdb3eb9c50e7b0489e9d2 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Mon, 16 Oct 2017 13:51:17 -0700 Subject: [PATCH 09/19] Improve docs, drop use of overloaded 'state'. 'State' has a specific meaning in IMP, so we shouldn't confuse the issue by referring to the 'previous state' of a Provenance decorator (the decorator should really be thought of as the transformation between states of the system anyway, not the state itself). --- modules/atom/include/provenance.h | 38 ++++++++++++++++------------ modules/atom/src/provenance.cpp | 10 ++++---- modules/atom/test/test_provenance.py | 8 +++--- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h index 947eb5dce0..632a53b56d 100644 --- a/modules/atom/include/provenance.h +++ b/modules/atom/include/provenance.h @@ -29,22 +29,22 @@ IMPATOM_BEGIN_NAMESPACE */ class IMPATOMEXPORT Provenance : public Decorator { static void do_setup_particle(Model *m, ParticleIndex pi) { - // Use self-index to indicate no previous state is set yet - m->add_attribute(get_previous_state_key(), pi, pi); + // Use self-index to indicate no previous provenance is set yet + m->add_attribute(get_previous_key(), pi, pi); } - static ParticleIndexKey get_previous_state_key(); + static ParticleIndexKey get_previous_key(); public: static bool get_is_setup(Model *m, ParticleIndex pi) { - return m->get_has_attribute(get_previous_state_key(), pi); + return m->get_has_attribute(get_previous_key(), pi); } - //! \return the previous state, or Provenance() if none exists. - Provenance get_previous_state() const { - ParticleIndex pi = get_model()->get_attribute(get_previous_state_key(), + //! \return the previous provenance, or Provenance() if none exists. + Provenance get_previous() const { + ParticleIndex pi = get_model()->get_attribute(get_previous_key(), get_particle_index()); - // self-index indicates no previous state is set yet + // self-index indicates no previous provenance is set yet if (pi == get_particle_index()) { return Provenance(); } else { @@ -52,14 +52,20 @@ class IMPATOMEXPORT Provenance : public Decorator { } } - //! Set the previous state - /** It is considered an error to try to set this more than once. */ - void set_previous_state(Provenance p) { - IMP_USAGE_CHECK(get_model()->get_attribute(get_previous_state_key(), + //! Set the previous provenance. + /** This can be used to show that a given part of the system was + generated through multiple steps in order, for example by first + being read from a PDB file, then sampled, filtered, and finally + clustered. + + \note it is considered an error to try to set this more than once. + */ + void set_previous(Provenance p) { + IMP_USAGE_CHECK(get_model()->get_attribute(get_previous_key(), get_particle_index()) == get_particle_index(), - "Previous state is already set"); - get_model()->set_attribute(get_previous_state_key(), + "Previous provenance is already set"); + get_model()->set_attribute(get_previous_key(), get_particle_index(), p.get_particle_index()); } @@ -188,7 +194,7 @@ class IMPATOMEXPORT SampleProvenance : public Provenance { /** Part of the system (usually the top of a Hierarchy) tagged with this decorator is understood to be a single frame within an ensemble that was created by combining a number of independent runs. One of those runs - should be the previous state of this provenance. The runs should be + should be the 'previous' provenance. The runs should be essentially identical, differing at most only in the number of frames. The total size of the resulting ensemble is stored here. */ @@ -237,7 +243,7 @@ class IMPATOMEXPORT CombineProvenance : public Provenance { //! Track creation of a system fragment by filtering. /** Part of the system (usually the top of a Hierarchy) tagged with this decorator is understood to be a single frame within an ensemble that - resulted from filtering a larger ensemble (the previous state of this + resulted from filtering a larger ensemble (the 'previous' provenance) by discarding models with scores above the threshold. */ class IMPATOMEXPORT FilterProvenance : public Provenance { diff --git a/modules/atom/src/provenance.cpp b/modules/atom/src/provenance.cpp index 407d6ca18b..ef44994e54 100644 --- a/modules/atom/src/provenance.cpp +++ b/modules/atom/src/provenance.cpp @@ -18,9 +18,9 @@ void Provenanced::show(std::ostream &out) const { out << "Provenanced" << std::endl; } -ParticleIndexKey Provenance::get_previous_state_key() { - static const ParticleIndexKey previous_state("previous_state"); - return previous_state; +ParticleIndexKey Provenance::get_previous_key() { + static const ParticleIndexKey previous("previous_provenance"); + return previous; } void Provenance::show(std::ostream &out) const { @@ -115,9 +115,9 @@ void FilterProvenance::show(std::ostream &out) const { void add_provenance(Model *m, ParticleIndex pi, Provenance p) { if (Provenanced::get_is_setup(m, pi)) { // add the new provenance as a new root - Provenanced pd = Provenanced(m, pi); + Provenanced pd(m, pi); Provenance old_provenance = pd.get_provenance(); - p.set_previous_state(old_provenance); + p.set_previous(old_provenance); pd.set_provenance(p); } else { Provenanced::setup_particle(m, pi, p); diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py index 3b06f70f20..0a8627517f 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/atom/test/test_provenance.py @@ -10,11 +10,11 @@ def test_provenance(self): m = IMP.Model() p = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) self.assertTrue(IMP.atom.Provenance.get_is_setup(p)) - self.assertFalse(p.get_previous_state()) + self.assertFalse(p.get_previous()) p2 = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) - p.set_previous_state(p2) - self.assertEqual(p.get_previous_state(), p2) + p.set_previous(p2) + self.assertEqual(p.get_previous(), p2) def test_structure_provenance(self): """Test StructureProvenance decorator""" @@ -105,7 +105,7 @@ def test_add_provenance(self): IMP.atom.add_provenance(m, p, prov2) self.assertTrue(IMP.atom.Provenanced.get_is_setup(p)) self.assertTrue(pd.get_provenance(), prov2) - self.assertTrue(pd.get_provenance().get_previous_state(), prov1) + self.assertTrue(pd.get_provenance().get_previous(), prov1) if __name__ == '__main__': IMP.test.main() From 98f776e24eae3684941dd2ef9fd69553926963dd Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 17 Oct 2017 11:44:33 -0700 Subject: [PATCH 10/19] Squashed 'modules/rmf/dependency/RMF/' changes from ea255a4453..8e5e39b171 8e5e39b171 Use nicer names for provenance attributes. 254ca59af7 We don't need a separate Provenance decorator. 04d7e37fe3 Add a new node type for provenance nodes. 3e94522e4d Add decorators to track provenance. 3dd7d083dd Fix to work with system Python. ab17fad084 Allow assigning a sequence to Chain nodes. e46d2a5f93 Add missing RMF_ENABLE_WARNINGS. git-subtree-dir: modules/rmf/dependency/RMF git-subtree-split: 8e5e39b1719eef8111689641f7fbaaa48d4a2aad --- modules/rmf/dependency/RMF/.travis.yml | 4 +- modules/rmf/dependency/RMF/ChangeLog.md | 2 + .../RMF/doc/DecoratorsAndAttributes.md | 32 +++++++- .../rmf/dependency/RMF/doc/MappingToRMF.md | 2 +- .../dependency/RMF/include/RMF/decorators.h | 1 + .../rmf/dependency/RMF/include/RMF/enums.h | 2 + .../RMF/include/RMF/show_hierarchy.h | 2 + modules/rmf/dependency/RMF/src/enums.cpp | 1 + .../rmf/dependency/RMF/src/show_hierarchy.cpp | 31 +++++++- modules/rmf/dependency/RMF/src/signature.cpp | 1 + .../rmf/dependency/RMF/swig/RMF.decorator.i | 1 + .../dependency/RMF/test/test_provenance.py | 78 +++++++++++++++++++ .../dependency/RMF/tools/build/_decorators.py | 37 +++++---- .../RMF/tools/build/make_decorators.py | 41 +++++++++- 14 files changed, 212 insertions(+), 23 deletions(-) create mode 100644 modules/rmf/dependency/RMF/test/test_provenance.py diff --git a/modules/rmf/dependency/RMF/.travis.yml b/modules/rmf/dependency/RMF/.travis.yml index b2cd003532..ba849a0e2d 100644 --- a/modules/rmf/dependency/RMF/.travis.yml +++ b/modules/rmf/dependency/RMF/.travis.yml @@ -26,9 +26,7 @@ script: - mkdir build - cd build - ../tools/coverage/setup.py - - PYTHON_INC=$(echo $(dirname $(which python))/../include/*/) - - PYTHON_LIB=$(echo /opt/python/2.7.*/lib/libpython2.7*.so) - - PYTHONPATH=`pwd`/coverage cmake .. -DCMAKE_BUILD_TYPE="$BUILD" -DCMAKE_CXX_FLAGS="$FLAGS" -DCMAKE_EXE_LINKER_FLAGS="$FLAGS" -DCMAKE_MODULE_LINKER_FLAGS="$FLAGS" -DCMAKE_SHARED_LINKER_FLAGS="$FLAGS" -DIMP_TEST_SETUP=$TEST_SETUP -DPYTHON_INCLUDE_DIR=$PYTHON_INC -DPYTHON_LIBRARY=$PYTHON_LIB + - PYTHONPATH=`pwd`/coverage cmake .. -DCMAKE_BUILD_TYPE="$BUILD" -DCMAKE_CXX_FLAGS="$FLAGS" -DCMAKE_EXE_LINKER_FLAGS="$FLAGS" -DCMAKE_MODULE_LINKER_FLAGS="$FLAGS" -DCMAKE_SHARED_LINKER_FLAGS="$FLAGS" -DIMP_TEST_SETUP=$TEST_SETUP - make -j 2 - export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so - ctest -j 2 --output-on-failure -L $TESTS diff --git a/modules/rmf/dependency/RMF/ChangeLog.md b/modules/rmf/dependency/RMF/ChangeLog.md index 0fab052ae9..fa5227b1b2 100644 --- a/modules/rmf/dependency/RMF/ChangeLog.md +++ b/modules/rmf/dependency/RMF/ChangeLog.md @@ -1,6 +1,8 @@ Change Log {#changelog} ========== +- A new category of decorators 'provenance' allows information about how the + structure was generated to be added to the file. - The new RMF::decorator::Reference decorator allows for a node to mark another node as its reference. - The new RMF::decorator::ExplicitResolution decorator allows attaching an diff --git a/modules/rmf/dependency/RMF/doc/DecoratorsAndAttributes.md b/modules/rmf/dependency/RMF/doc/DecoratorsAndAttributes.md index c06f3844da..98048e1e91 100644 --- a/modules/rmf/dependency/RMF/doc/DecoratorsAndAttributes.md +++ b/modules/rmf/dependency/RMF/doc/DecoratorsAndAttributes.md @@ -77,7 +77,7 @@ The category name is `sequence` and it includes information about the types and | Name | Node Type | Attributes | |--------------------------------:|:-----------------------:|:-----------------------------------| | RMF::decorator::Residue | RMF::REPRESENTATION | index, residue type | -| RMF::decorator::Chain | RMF::REPRESENTATION | chain id | +| RMF::decorator::Chain | RMF::REPRESENTATION | chain id, sequence | | RMF::decorator::Domain | RMF::REPRESENTATION | first residue index, last residue index | | RMF::decorator::Fragment | RMF::REPRESENTATION | residue indexes | | RMF::decorator::Copy | RMF::REPRESENTATION | copy index | @@ -207,3 +207,33 @@ should be loaded as a child of the current node. | Name | Node Type | Attributes | |--------------------------------:|:-----------------------:|:-----------------------------------| | RMF::decorator::JournalArticle | RMF::ORGANIZATIONAL | title, journal, pubmed id, year, authors | + +# Provenance # {#provenance} + +The category name is `provenance`. It includes information about how the +structure was generated. + +## Attributes ## {#provenanceattributes} + +| Name | Type | Description | +|----------------------:|-------------:|:------------------------------------------| +| `structure filename` | string | File from which the structure was read | +| `structure chain` | string | Chain ID of the structure that was read | +| `sampling method` | string | Sampling method utilized | +| `sampling frames` | int | Number of frames in the sample ensemble | +| `sampling iterations` | int | Number of sampling iterations used | +| `combined runs` | int | Number of sampling runs utilized | +| `combined frames` | int | Total number of frames combined | +| `filter threshold` | float | Score threshold to discard bad models | +| `cluster members` | int | Number of members in a cluster | + + +## Decorators ## {#provenancedecorators} + +| Name | Node Type | Attributes | +|------------------------------------:|:-------------------:|:------------------------------------------------------| +| RMF::decorator::StructureProvenance | RMF::PROVENANCE | structure filename, structure chain | +| RMF::decorator::SampleProvenance | RMF::PROVENANCE | sampling method, sampling frames, sampling iterations | +| RMF::decorator::CombineProvenance | RMF::PROVENANCE | combined runs, combined frames | +| RMF::decorator::FilterProvenance | RMF::PROVENANCE | filter threshold, filter frames | +| RMF::decorator::ClusterProvenance | RMF::PROVENANCE | cluster members | diff --git a/modules/rmf/dependency/RMF/doc/MappingToRMF.md b/modules/rmf/dependency/RMF/doc/MappingToRMF.md index eaa091ce6b..b9ccb9512a 100644 --- a/modules/rmf/dependency/RMF/doc/MappingToRMF.md +++ b/modules/rmf/dependency/RMF/doc/MappingToRMF.md @@ -15,7 +15,7 @@ For nodes types, the following node shapes are used } \enddot And a dot means an RMF::ALIAS node. The key concept to keep in mind with RMF::ALIAS nodes is that, unlike everything else, they don't introduce a new entity, simply point to one that exists elsewhere. -See RMF::ROOT, RMF::REPRESENTATION, RMF::BOND, RMF::ORGANIZATIONAL, RMF::FEATURE for more information. +See RMF::ROOT, RMF::REPRESENTATION, RMF::BOND, RMF::ORGANIZATIONAL, RMF::PROVENANCE, RMF::FEATURE for more information. # PDB files # diff --git a/modules/rmf/dependency/RMF/include/RMF/decorators.h b/modules/rmf/dependency/RMF/include/RMF/decorators.h index de8d0e2007..b18db3e77e 100644 --- a/modules/rmf/dependency/RMF/include/RMF/decorators.h +++ b/modules/rmf/dependency/RMF/include/RMF/decorators.h @@ -16,6 +16,7 @@ #include "RMF/decorator/feature.h" #include "RMF/decorator/physics.h" #include "RMF/decorator/publication.h" +#include "RMF/decorator/provenance.h" #include "RMF/decorator/sequence.h" #include "RMF/decorator/shape.h" #include "RMF/decorator/bond.h" diff --git a/modules/rmf/dependency/RMF/include/RMF/enums.h b/modules/rmf/dependency/RMF/include/RMF/enums.h index 926486b681..22020ad424 100644 --- a/modules/rmf/dependency/RMF/include/RMF/enums.h +++ b/modules/rmf/dependency/RMF/include/RMF/enums.h @@ -57,6 +57,8 @@ extern RMFEXPORT const NodeType BOND; /** This includes nodes that are just RMF::ReferenceFrame nodes. */ extern RMFEXPORT const NodeType ORGANIZATIONAL; +//! Represent the process by which a structure was created +extern RMFEXPORT const NodeType PROVENANCE; #ifndef RMF_DOXYGEN //! An internal link to another node extern RMFEXPORT const NodeType LINK; diff --git a/modules/rmf/dependency/RMF/include/RMF/show_hierarchy.h b/modules/rmf/dependency/RMF/include/RMF/show_hierarchy.h index 8b1aaade15..ddccfd4705 100644 --- a/modules/rmf/dependency/RMF/include/RMF/show_hierarchy.h +++ b/modules/rmf/dependency/RMF/include/RMF/show_hierarchy.h @@ -12,6 +12,8 @@ #include #include +RMF_ENABLE_WARNINGS + namespace RMF { class NodeConstHandle; class FileConstHandle; diff --git a/modules/rmf/dependency/RMF/src/enums.cpp b/modules/rmf/dependency/RMF/src/enums.cpp index d9fdb1ae78..61a83f4a0e 100644 --- a/modules/rmf/dependency/RMF/src/enums.cpp +++ b/modules/rmf/dependency/RMF/src/enums.cpp @@ -24,6 +24,7 @@ const NodeType CUSTOM(5, "custom"); const NodeType BOND(6, "bond"); const NodeType ORGANIZATIONAL(7, "organizational"); const NodeType LINK(8, "link"); +const NodeType PROVENANCE(9, "provenance"); const FrameType INVALID_FRAME_TYPE(-1, "inv"); const FrameType STATIC(0, "static"); diff --git a/modules/rmf/dependency/RMF/src/show_hierarchy.cpp b/modules/rmf/dependency/RMF/src/show_hierarchy.cpp index 74aacfc810..3fee6acb08 100644 --- a/modules/rmf/dependency/RMF/src/show_hierarchy.cpp +++ b/modules/rmf/dependency/RMF/src/show_hierarchy.cpp @@ -18,6 +18,7 @@ #include "RMF/decorator/alternatives.h" #include "RMF/decorator/shape.h" #include "RMF/decorator/reference.h" +#include "RMF/decorator/provenance.h" #include "RMF/enums.h" #include "RMF/infrastructure_macros.h" #include "RMF/types.h" @@ -175,6 +176,11 @@ void show_node_decorators( decorator::ChainFactory chaincf, decorator::DomainFactory fragcf, decorator::CopyFactory copycf, decorator::DiffuserFactory diffusercf, decorator::TypedFactory typedcf, decorator::ReferenceFactory refcf, + decorator::StructureProvenanceFactory strucpcf, + decorator::SampleProvenanceFactory samppcf, + decorator::CombineProvenanceFactory combpcf, + decorator::FilterProvenanceFactory filtpcf, + decorator::ClusterProvenanceFactory clustpcf, std::string) { using std::operator<<; out << "\"" << n.get_name() << "\"" << node_suffix << " [" << n.get_type() @@ -219,6 +225,16 @@ void show_node_decorators( else if (diffusercf.get_is(n)) out << " diffuser"; if (refcf.get_is_static(n)) out << " reference(s)"; else if (refcf.get_is(n)) out << " reference"; + if (strucpcf.get_is_static(n)) out << " structure provenance(s)"; + else if (strucpcf.get_is(n)) out << " structure provenance"; + if (samppcf.get_is_static(n)) out << " sample provenance(s)"; + else if (samppcf.get_is(n)) out << " sample provenance"; + if (combpcf.get_is_static(n)) out << " combine provenance(s)"; + else if (combpcf.get_is(n)) out << " combine provenance"; + if (filtpcf.get_is_static(n)) out << " filter provenance(s)"; + else if (filtpcf.get_is(n)) out << " filter provenance"; + if (clustpcf.get_is_static(n)) out << " cluster provenance(s)"; + else if (clustpcf.get_is(n)) out << " cluster provenance"; out << "]"; } @@ -278,6 +294,11 @@ struct ShowDecorators { decorator::DiffuserFactory diffusercf; decorator::TypedFactory typedcf; decorator::ReferenceFactory refcf; + decorator::StructureProvenanceFactory strucpcf; + decorator::SampleProvenanceFactory samppcf; + decorator::CombineProvenanceFactory combpcf; + decorator::FilterProvenanceFactory filtpcf; + decorator::ClusterProvenanceFactory clustpcf; ShowDecorators(FileConstHandle fh) : bdf(fh), ccf(fh), @@ -296,12 +317,18 @@ struct ShowDecorators { copycf(fh), diffusercf(fh), typedcf(fh), - refcf(fh) {} + refcf(fh), + strucpcf(fh), + samppcf(fh), + combpcf(fh), + filtpcf(fh), + clustpcf(fh) {} void operator()(NodeConstHandle cur, std::string prefix, std::string suffix, std::ostream& out) { show_node_decorators(cur, suffix, out, bdf, ccf, pcf, ipcf, rpcf, scf, repcf, bcf, cycf, segcf, rcf, acf, chaincf, fragcf, - copycf, diffusercf, typedcf, refcf, prefix + " "); + copycf, diffusercf, typedcf, refcf, strucpcf, + samppcf, combpcf, filtpcf, clustpcf, prefix + " "); } }; } diff --git a/modules/rmf/dependency/RMF/src/signature.cpp b/modules/rmf/dependency/RMF/src/signature.cpp index acb167c740..b1cce897de 100644 --- a/modules/rmf/dependency/RMF/src/signature.cpp +++ b/modules/rmf/dependency/RMF/src/signature.cpp @@ -20,6 +20,7 @@ #include "RMF/decorator/feature.h" #include "RMF/decorator/bond.h" #include "RMF/decorator/reference.h" +#include "RMF/decorator/provenance.h" #include "RMF/decorator/shape.h" RMF_ENABLE_WARNINGS diff --git a/modules/rmf/dependency/RMF/swig/RMF.decorator.i b/modules/rmf/dependency/RMF/swig/RMF.decorator.i index 4d16584c8d..568290cdde 100644 --- a/modules/rmf/dependency/RMF/swig/RMF.decorator.i +++ b/modules/rmf/dependency/RMF/swig/RMF.decorator.i @@ -10,3 +10,4 @@ %include "RMF/decorator/bond.h" %include "RMF/decorator/labels.h" %include "RMF/decorator/reference.h" +%include "RMF/decorator/provenance.h" diff --git a/modules/rmf/dependency/RMF/test/test_provenance.py b/modules/rmf/dependency/RMF/test/test_provenance.py new file mode 100644 index 0000000000..7f6e0ff6dd --- /dev/null +++ b/modules/rmf/dependency/RMF/test/test_provenance.py @@ -0,0 +1,78 @@ +from __future__ import print_function +import sys +import RMF +import unittest +import os + +class Tests(unittest.TestCase): + + def test_provenance(self): + """Test the Provenance decorator""" + for suffix in RMF.suffixes: + fname = RMF._get_temporary_file_path("provenance." + suffix) + self._create(fname) + self._read(fname) + + def _add_provenance_nodes(self, rmf, rt): + """Add *Provenance nodes under rt. Return the root.""" + strucpf = RMF.StructureProvenanceFactory(rmf) + samppf = RMF.SampleProvenanceFactory(rmf) + clustpf = RMF.ClusterProvenanceFactory(rmf) + + clust_node = rt.add_child("clustering", RMF.PROVENANCE) + clust = clustpf.get(clust_node) + clust.set_members(10) + + samp_node = clust_node.add_child("sampling", RMF.PROVENANCE) + samp = samppf.get(samp_node) + samp.set_method("Monte Carlo") + samp.set_frames(100) + samp.set_iterations(10) + + struc_node = samp_node.add_child("structure", RMF.PROVENANCE) + struc = strucpf.get(struc_node) + struc.set_filename('foo.pdb') + struc.set_chain('X') + + return clust_node + + def _create(self, fname): + rmf = RMF.create_rmf_file(fname) + rmf.add_frame('zero', RMF.FRAME) + rt = rmf.get_root_node() + + c1 = rt.add_child("c1", RMF.REPRESENTATION) + c0 = self._add_provenance_nodes(rmf, c1) + self._check_provenance_nodes(rmf, c0) + + def _read(self, fname): + rmf = RMF.open_rmf_file_read_only(fname) + rt = rmf.get_root_node() + c1, = rt.get_children() + c0, = c1.get_children() + self._check_provenance_nodes(rmf, c0) + + def _check_provenance_nodes(self, rmf, prov_root): + strucpf = RMF.StructureProvenanceFactory(rmf) + samppf = RMF.SampleProvenanceFactory(rmf) + clustpf = RMF.ClusterProvenanceFactory(rmf) + + self.assertTrue(clustpf.get_is(prov_root)) + clust = clustpf.get(prov_root) + self.assertEqual(clust.get_members(), 10) + + samp_node = prov_root.get_children()[0] + self.assertTrue(samppf.get_is(samp_node)) + samp = samppf.get(samp_node) + self.assertEqual(samp.get_frames(), 100) + self.assertEqual(samp.get_iterations(), 10) + self.assertEqual(samp.get_method(), "Monte Carlo") + + struc_node = samp_node.get_children()[0] + self.assertTrue(strucpf.get_is(struc_node)) + struc = strucpf.get(struc_node) + self.assertEqual(struc.get_filename(), 'foo.pdb') + self.assertEqual(struc.get_chain(), 'X') + +if __name__ == '__main__': + unittest.main() diff --git a/modules/rmf/dependency/RMF/tools/build/_decorators.py b/modules/rmf/dependency/RMF/tools/build/_decorators.py index cef4f333f0..defb3376f4 100755 --- a/modules/rmf/dependency/RMF/tools/build/_decorators.py +++ b/modules/rmf/dependency/RMF/tools/build/_decorators.py @@ -72,13 +72,12 @@ def get_check(self): class Attribute(Base): - def __init__( - self, - name, - attribute_type, - function_name=None): + def __init__(self, name, attribute_type, function_name=None, + allow_null=False): if not function_name: - function_name = name.replace(" ", "_") + self.function_name = name.replace(" ", "_") + else: + self.function_name = function_name Base.__init__(self, name, attribute_type + "Key", attribute_type) self.get_methods = """ @@ -97,7 +96,11 @@ def __init__( return get_node().GET_STATIC(NAME_); } RMF_DECORATOR_CATCH( ); } -""" % (function_name, function_name, function_name) +""" % (self.function_name, self.function_name, self.function_name) + # Note that this currently only works for string attributes + if allow_null: + self.get_methods = self.get_methods.replace('return', + 'if (!get_node().get_has_value(NAME_)) return "";\nreturn') self.set_methods = """ void set_%s(TYPE v) { try { @@ -114,8 +117,12 @@ def __init__( get_node().SET_STATIC(NAME_, v); } RMF_DECORATOR_CATCH( ); } -""" % (function_name, function_name, function_name) - self.check = "!nh.GET(NAME_).get_is_null()" +""" % (self.function_name, self.function_name, self.function_name) + # If the attribute is allowed to be null, skip check + if allow_null: + self.check = "" + else: + self.check = "!nh.GET(NAME_).get_is_null()" self.data_initialize = "fh.get_key(cat_, \"%s\")" % name @@ -142,26 +149,26 @@ def __init__(self, name): class PathAttribute(Attribute): - def __init__(self, name): - Attribute.__init__(self, name, "String") + def __init__(self, name, function_name=None): + Attribute.__init__(self, name, "String", function_name) self.get_methods = """ - String get_NAME() const { + String get_%s() const { try { String relpath = get_node().GET_BOTH(NAME_); String filename = get_node().get_file().get_path(); return internal::get_absolute_path(filename, relpath); } RMF_DECORATOR_CATCH( ); } -""" +""" % self.function_name self.set_methods = """ - void set_NAME(String path) { + void set_%s(String path) { try { String filename = get_node().get_file().get_path(); String relpath = internal::get_relative_path(filename, path); get_node().SET_BOTH(NAME_, relpath); } RMF_DECORATOR_CATCH( ); } -""" +""" % self.function_name class AttributePair(Base): diff --git a/modules/rmf/dependency/RMF/tools/build/make_decorators.py b/modules/rmf/dependency/RMF/tools/build/make_decorators.py index ea16f8efd2..74480a3699 100755 --- a/modules/rmf/dependency/RMF/tools/build/make_decorators.py +++ b/modules/rmf/dependency/RMF/tools/build/make_decorators.py @@ -103,6 +103,44 @@ make_header("publication", [journal], []) +structure = Decorator(["PROVENANCE"], "provenance", + "StructureProvenance", + # Note that this should really be PathAttribute, + # but that currently requires that the file exists, + # otherwise reading the RMF file will fail + [Attribute("structure filename", "String", + function_name='filename'), + Attribute("structure chain", "String", + function_name='chain')]) + +sample = Decorator(["PROVENANCE"], "provenance", + "SampleProvenance", + [Attribute("sampling method", "String", + function_name='method'), + Attribute("sampling frames", "Int", + function_name='frames'), + Attribute("sampling iterations", "Int", + function_name='iterations')]) + +combine = Decorator(["PROVENANCE"], "provenance", + "CombineProvenance", + [Attribute("combined runs", "Int", function_name='runs'), + Attribute("combined frames", "Int", + function_name='frames')]) + +filterp = Decorator(["PROVENANCE"], "provenance", + "FilterProvenance", + [Attribute("filter threshold", "Float", + function_name='threshold'), + Attribute("filter frames", "Int", function_name='frames')]) + +cluster = Decorator(["PROVENANCE"], "provenance", + "ClusterProvenance", + [Attribute("cluster members", "Int", + function_name='members')]) + +make_header("provenance", + [structure, sample, combine, filterp, cluster], []) residue = Decorator(["REPRESENTATION"], "sequence", "Residue", @@ -111,7 +149,8 @@ chain = Decorator(["REPRESENTATION"], "sequence", "Chain", - [Attribute("chain id", "String")]) + [Attribute("chain id", "String"), + Attribute("sequence", "String", allow_null=True)]) domain = Decorator(["REPRESENTATION"], "sequence", "Domain", From d66763ed44acb2c90b8d95a470367e10183de46d Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 17 Oct 2017 15:14:52 -0700 Subject: [PATCH 11/19] Be sure to also set up the base class. --- modules/atom/include/provenance.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/atom/include/provenance.h b/modules/atom/include/provenance.h index 632a53b56d..9684ebca9a 100644 --- a/modules/atom/include/provenance.h +++ b/modules/atom/include/provenance.h @@ -78,6 +78,7 @@ class IMPATOMEXPORT StructureProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, std::string filename, std::string chain_id) { + Provenance::setup_particle(m, pi); IMP_USAGE_CHECK(!filename.empty(), "The filename cannot be empty."); m->add_attribute(get_filename_key(), pi, filename); m->add_attribute(get_chain_key(), pi, chain_id); @@ -131,6 +132,7 @@ class IMPATOMEXPORT StructureProvenance : public Provenance { class IMPATOMEXPORT SampleProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, std::string method, int frames) { + Provenance::setup_particle(m, pi); m->add_attribute(get_method_key(), pi, method); m->add_attribute(get_frames_key(), pi, frames); m->add_attribute(get_iterations_key(), pi, 1); @@ -201,6 +203,7 @@ class IMPATOMEXPORT SampleProvenance : public Provenance { class IMPATOMEXPORT CombineProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, int runs, int frames) { + Provenance::setup_particle(m, pi); m->add_attribute(get_runs_key(), pi, runs); m->add_attribute(get_frames_key(), pi, frames); } @@ -249,6 +252,7 @@ class IMPATOMEXPORT CombineProvenance : public Provenance { class IMPATOMEXPORT FilterProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, double threshold, int frames) { + Provenance::setup_particle(m, pi); m->add_attribute(get_threshold_key(), pi, threshold); m->add_attribute(get_frames_key(), pi, frames); } @@ -297,6 +301,7 @@ class IMPATOMEXPORT FilterProvenance : public Provenance { */ class IMPATOMEXPORT ClusterProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, int members) { + Provenance::setup_particle(m, pi); m->add_attribute(get_members_key(), pi, members); } From 49d1f7f6b1dde9bcfbaf7090ce5f04b2b04e07fe Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 17 Oct 2017 15:15:19 -0700 Subject: [PATCH 12/19] Add more stringent checks. --- modules/atom/test/test_provenance.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/atom/test/test_provenance.py b/modules/atom/test/test_provenance.py index 0a8627517f..60f6ea0dda 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/atom/test/test_provenance.py @@ -84,7 +84,9 @@ def test_provenanced(self): pd = IMP.atom.Provenanced.setup_particle(m, IMP.Particle(m), p) self.assertEqual(pd.get_provenance(), p) pd.set_provenance(p) + self.assertEqual(pd.get_provenance(), p) self.assertTrue(IMP.atom.Provenanced.get_is_setup(pd)) + self.assertTrue(IMP.atom.Provenance.get_is_setup(p)) self.assertFalse(IMP.atom.Provenanced.get_is_setup(p)) self.assertFalse(IMP.atom.Provenance.get_is_setup(pd)) From e3217ee725aec081d4198175bc0c0c2338a8c16a Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 17 Oct 2017 15:18:35 -0700 Subject: [PATCH 13/19] Copy provenance info to and from RMF files. --- modules/rmf/include/atom_links.h | 24 +++++ modules/rmf/src/atom_links.cpp | 132 +++++++++++++++++++++++++++- modules/rmf/test/test_provenance.py | 103 ++++++++++++++++++++++ 3 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 modules/rmf/test/test_provenance.py diff --git a/modules/rmf/include/atom_links.h b/modules/rmf/include/atom_links.h index e1d27abb8b..c2262e4498 100644 --- a/modules/rmf/include/atom_links.h +++ b/modules/rmf/include/atom_links.h @@ -16,11 +16,13 @@ #include "internal/atom_links_xyzs.h" #include "internal/atom_links_gaussians.h" #include +#include #include #include #include #include #include +#include #include #include #include @@ -37,6 +39,11 @@ class IMPRMFEXPORT HierarchyLoadLink : public SimpleLoadLink { RMF::decorator::IntermediateParticleFactory intermediate_particle_factory_; RMF::decorator::ReferenceFrameFactory reference_frame_factory_; RMF::decorator::AlternativesFactory af_; + RMF::decorator::StructureProvenanceFactory strucpf_; + RMF::decorator::SampleProvenanceFactory samppf_; + RMF::decorator::CombineProvenanceFactory combpf_; + RMF::decorator::FilterProvenanceFactory filtpf_; + RMF::decorator::ClusterProvenanceFactory clustpf_; RMF::decorator::ExplicitResolutionFactory explicit_resolution_factory_; RMF::IntKey external_rigid_body_key_; struct Data { @@ -72,6 +79,14 @@ class IMPRMFEXPORT HierarchyLoadLink : public SimpleLoadLink { void create_recursive(Model *m, ParticleIndex root, ParticleIndex cur, RMF::NodeConstHandle name, ParticleIndexes rigid_bodies, Data &data); + + // Make tree of *Provenance IMP particles corresponding to those in the RMF + void create_provenance(Model *m, RMF::NodeConstHandle node, + ParticleIndex cur); + + // Make *Provenance IMP particle corresponding to that in the RMF + atom::Provenance create_one_provenance(Model *m, RMF::NodeConstHandle node); + virtual void do_load_one(RMF::NodeConstHandle nh, Particle *o) IMP_FINAL IMP_OVERRIDE; @@ -138,6 +153,11 @@ class IMPRMFEXPORT HierarchySaveLink : public SimpleSaveLink { DM; DM data_; RMF::decorator::AlternativesFactory af_; + RMF::decorator::StructureProvenanceFactory strucpf_; + RMF::decorator::SampleProvenanceFactory samppf_; + RMF::decorator::CombineProvenanceFactory combpf_; + RMF::decorator::FilterProvenanceFactory filtpf_; + RMF::decorator::ClusterProvenanceFactory clustpf_; RMF::decorator::ExplicitResolutionFactory explicit_resolution_factory_; RMF::IntKey external_rigid_body_key_; @@ -145,6 +165,10 @@ class IMPRMFEXPORT HierarchySaveLink : public SimpleSaveLink { ParticleIndex p, ParticleIndexes rigid_bodies, RMF::NodeHandle cur, Data &data); + + // Make RMF PROVENANCE nodes corresponding to those in IMP + void add_provenance(Model *m, ParticleIndex p, RMF::NodeHandle cur); + virtual void do_add(Particle *p, RMF::NodeHandle cur) IMP_OVERRIDE; virtual void do_save_one(Particle *o, RMF::NodeHandle nh) IMP_OVERRIDE; diff --git a/modules/rmf/src/atom_links.cpp b/modules/rmf/src/atom_links.cpp index 5f5d29f574..0f92fb0dab 100644 --- a/modules/rmf/src/atom_links.cpp +++ b/modules/rmf/src/atom_links.cpp @@ -40,8 +40,28 @@ std::string get_good_name(Model *m, ParticleIndex h) { return m->get_particle_name(h); } } + +// Get the 'child' provenance from the RMF, or a default-constructed +// object if no children exist +RMF::NodeConstHandle get_previous_rmf_provenance(RMF::NodeConstHandle node) { + RMF::NodeConstHandles nchs; + IMP_FOREACH(RMF::NodeConstHandle ch, node.get_children()) { + if (ch.get_type() == RMF::PROVENANCE) { + nchs.push_back(ch); + } + } + if (nchs.size() > 1) { + IMP_THROW("RMF provenance hierarchy has more than one child at " << node, + IOException); + } else if (nchs.empty()) { + return RMF::NodeConstHandle(); + } else { + return nchs[0]; + } } +} // anonymous namespace + void HierarchyLoadLink::do_load_one(RMF::NodeConstHandle nh, Particle *o) { data_.find(o->get_index()) @@ -118,6 +138,11 @@ void HierarchyLoadLink::create_recursive(Model *m, atom::Hierarchy(m, cur) .add_child(atom::Hierarchy::setup_particle(m, child)); } + } else if (ch.get_type() == RMF::PROVENANCE) { + // Note that at most only one such node should be encountered. If more + // than one is found, this is an error (and Provenanced::setup_particle() + // will throw) + create_provenance(m, ch, cur); } } do_setup_particle(m, root, cur, name); @@ -143,6 +168,53 @@ void HierarchyLoadLink::create_recursive(Model *m, } } +// Make *Provenance IMP particles corresponding to those in the RMF +void HierarchyLoadLink::create_provenance(Model *m, RMF::NodeConstHandle node, + ParticleIndex cur) { + atom::Provenance prov = create_one_provenance(m, node); + atom::Provenanced provd = atom::Provenanced::setup_particle(m, cur, prov); + + while ((node = get_previous_rmf_provenance(node)) != RMF::NodeConstHandle()) { + atom::Provenance thisprov = create_one_provenance(m, node); + prov.set_previous(thisprov); + prov = thisprov; + } +} + +atom::Provenance HierarchyLoadLink::create_one_provenance(Model *m, + RMF::NodeConstHandle node) { + if (strucpf_.get_is(node)) { + RMF::decorator::StructureProvenanceConst rp = strucpf_.get(node); + ParticleIndex ip = m->add_particle(node.get_name()); + return atom::StructureProvenance::setup_particle(m, ip, rp.get_filename(), + rp.get_chain()); + } else if (samppf_.get_is(node)) { + RMF::decorator::SampleProvenanceConst rp = samppf_.get(node); + ParticleIndex ip = m->add_particle(node.get_name()); + atom::SampleProvenance sp = + atom::SampleProvenance::setup_particle(m, ip, rp.get_method(), + rp.get_frames()); + sp.set_number_of_iterations(rp.get_iterations()); + return sp; + } else if (combpf_.get_is(node)) { + RMF::decorator::CombineProvenanceConst rp = combpf_.get(node); + ParticleIndex ip = m->add_particle(node.get_name()); + return atom::CombineProvenance::setup_particle(m, ip, rp.get_runs(), + rp.get_frames()); + } else if (filtpf_.get_is(node)) { + RMF::decorator::FilterProvenanceConst rp = filtpf_.get(node); + ParticleIndex ip = m->add_particle(node.get_name()); + return atom::FilterProvenance::setup_particle(m, ip, rp.get_threshold(), + rp.get_frames()); + } else if (clustpf_.get_is(node)) { + RMF::decorator::ClusterProvenanceConst rp = clustpf_.get(node); + ParticleIndex ip = m->add_particle(node.get_name()); + return atom::ClusterProvenance::setup_particle(m, ip, rp.get_members()); + } else { + IMP_THROW("Unhandled provenance type " << node, IOException); + } +} + Particle *HierarchyLoadLink::do_create(RMF::NodeConstHandle node, Model *m) { IMP_FUNCTION_LOG; @@ -247,7 +319,8 @@ HierarchyLoadLink::HierarchyLoadLink(RMF::FileConstHandle fh) : P("HierarchyLoadLink%1%"), intermediate_particle_factory_(fh), reference_frame_factory_(fh), - af_(fh), + af_(fh), strucpf_(fh), samppf_(fh), + combpf_(fh), filtpf_(fh), clustpf_(fh), explicit_resolution_factory_(fh) { RMF::Category imp_cat = fh.get_category("IMP"); external_rigid_body_key_ = @@ -347,6 +420,10 @@ void HierarchySaveLink::add_recursive(Model *m, ParticleIndex root, } } + if (atom::Provenanced::get_is_setup(m, p)) { + add_provenance(m, p, cur); + } + if (!prep_nodes.empty() || !grep_nodes.empty()) { RMF::decorator::Alternatives ad = af_.get(cur); IMP_FOREACH(RMF::NodeHandle nh, prep_nodes) { @@ -358,8 +435,59 @@ void HierarchySaveLink::add_recursive(Model *m, ParticleIndex root, } } +// Make RMF PROVENANCE nodes corresponding to those in IMP +void HierarchySaveLink::add_provenance(Model *m, ParticleIndex p, + RMF::NodeHandle cur) { + atom::Provenanced provd(m, p); + for (atom::Provenance prov = provd.get_provenance(); prov; + prov = prov.get_previous()) { + if (atom::StructureProvenance::get_is_setup(prov)) { + atom::StructureProvenance ip(prov); + cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), + RMF::PROVENANCE); + RMF::decorator::StructureProvenance rp = strucpf_.get(cur); + rp.set_filename(ip.get_filename()); + rp.set_chain(ip.get_chain_id()); + } else if (atom::SampleProvenance::get_is_setup(prov)) { + atom::SampleProvenance ip(prov); + cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), + RMF::PROVENANCE); + RMF::decorator::SampleProvenance rp = samppf_.get(cur); + rp.set_method(ip.get_method()); + rp.set_frames(ip.get_number_of_frames()); + rp.set_iterations(ip.get_number_of_iterations()); + } else if (atom::CombineProvenance::get_is_setup(prov)) { + atom::CombineProvenance ip(prov); + cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), + RMF::PROVENANCE); + RMF::decorator::CombineProvenance rp = combpf_.get(cur); + rp.set_runs(ip.get_number_of_runs()); + rp.set_frames(ip.get_number_of_frames()); + } else if (atom::FilterProvenance::get_is_setup(prov)) { + atom::FilterProvenance ip(prov); + cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), + RMF::PROVENANCE); + RMF::decorator::FilterProvenance rp = filtpf_.get(cur); + rp.set_threshold(ip.get_threshold()); + rp.set_frames(ip.get_number_of_frames()); + } else if (atom::ClusterProvenance::get_is_setup(prov)) { + atom::ClusterProvenance ip(prov); + cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), + RMF::PROVENANCE); + RMF::decorator::ClusterProvenance rp = clustpf_.get(cur); + rp.set_members(ip.get_number_of_members()); + } else { + IMP_THROW("Unhandled provenance type " + << m->get_particle_name(prov.get_particle_index()), + IOException); + } + } +} + HierarchySaveLink::HierarchySaveLink(RMF::FileHandle f) - : P("HierarchySaveLink%1%"), af_(f), explicit_resolution_factory_(f) { + : P("HierarchySaveLink%1%"), af_(f), strucpf_(f), samppf_(f), + combpf_(f), filtpf_(f), clustpf_(f), + explicit_resolution_factory_(f) { RMF::Category imp_cat = f.get_category("IMP"); external_rigid_body_key_ = f.get_key(imp_cat, "external frame", RMF::IntTraits()); diff --git a/modules/rmf/test/test_provenance.py b/modules/rmf/test/test_provenance.py new file mode 100644 index 0000000000..892d195cfa --- /dev/null +++ b/modules/rmf/test/test_provenance.py @@ -0,0 +1,103 @@ +from __future__ import print_function +import unittest +import IMP.rmf +import IMP.test +import RMF + + +class Tests(IMP.test.TestCase): + + def add_provenance(self, h): + m = h.get_model() + struc = IMP.atom.StructureProvenance.setup_particle( + m, IMP.Particle(m), "testfile", "testchain") + struc.set_name("structure provenance") + IMP.atom.add_provenance(m, h, struc) + + samp = IMP.atom.SampleProvenance.setup_particle( + m, IMP.Particle(m), "Monte Carlo", 100) + samp.set_number_of_iterations(42) + IMP.atom.add_provenance(m, h, samp) + + comb = IMP.atom.CombineProvenance.setup_particle( + m, IMP.Particle(m), 4, 27) + IMP.atom.add_provenance(m, h, comb) + + filt = IMP.atom.FilterProvenance.setup_particle( + m, IMP.Particle(m), 100.5, 39) + IMP.atom.add_provenance(m, h, filt) + + clus = IMP.atom.ClusterProvenance.setup_particle(m, IMP.Particle(m), 10) + IMP.atom.add_provenance(m, h, clus) + + def check_provenance(self, h): + m = h.get_model() + + # Test IMP-added chain provenance + chain, = IMP.atom.get_by_type(h, IMP.atom.CHAIN_TYPE) + self.assertTrue(IMP.atom.Provenanced.get_is_setup(m, chain)) + prov = IMP.atom.Provenanced(m, chain).get_provenance() + self.assertTrue(IMP.atom.StructureProvenance.get_is_setup(m, prov)) + struc = IMP.atom.StructureProvenance(m, prov) + self.assertEqual(struc.get_chain_id(), 'A') + + # Should be no more chain provenance + prov = prov.get_previous() + self.assertFalse(prov) + + # Check the provenance we added at the top level + self.assertTrue(IMP.atom.Provenanced.get_is_setup(m, h)) + prov = IMP.atom.Provenanced(m, h).get_provenance() + + self.assertTrue(IMP.atom.ClusterProvenance.get_is_setup(m, prov)) + clus = IMP.atom.ClusterProvenance(m, prov) + self.assertEqual(clus.get_number_of_members(), 10) + + prov = prov.get_previous() + self.assertTrue(IMP.atom.FilterProvenance.get_is_setup(m, prov)) + filt = IMP.atom.FilterProvenance(m, prov) + self.assertAlmostEqual(filt.get_threshold(), 100.5, delta=1e-4) + self.assertEqual(filt.get_number_of_frames(), 39) + + prov = prov.get_previous() + self.assertTrue(IMP.atom.CombineProvenance.get_is_setup(m, prov)) + comb = IMP.atom.CombineProvenance(m, prov) + self.assertEqual(comb.get_number_of_runs(), 4) + self.assertEqual(comb.get_number_of_frames(), 27) + + prov = prov.get_previous() + self.assertTrue(IMP.atom.SampleProvenance.get_is_setup(m, prov)) + samp = IMP.atom.SampleProvenance(m, prov) + self.assertEqual(samp.get_method(), "Monte Carlo") + self.assertEqual(samp.get_number_of_frames(), 100) + self.assertEqual(samp.get_number_of_iterations(), 42) + + prov = prov.get_previous() + self.assertTrue(IMP.atom.StructureProvenance.get_is_setup(m, prov)) + struc = IMP.atom.StructureProvenance(m, prov) + self.assertEqual(struc.get_filename(), "testfile") + self.assertEqual(struc.get_chain_id(), "testchain") + self.assertEqual(struc.get_name(), "structure provenance") + + # Should be no more provenance + prov = prov.get_previous() + self.assertFalse(prov) + + def test_rt(self): + """Test that provenance info can be stored in RMF files""" + for suffix in [".rmfz", ".rmf3"]: + m = IMP.Model() + name = self.get_tmp_file_name("test_provenance" + suffix) + h = IMP.atom.read_pdb(self.get_input_file_name("simple.pdb"), m, + IMP.atom.NonAlternativePDBSelector()) + self.add_provenance(h) + f = RMF.create_rmf_file(name) + IMP.rmf.add_hierarchy(f, h) + IMP.rmf.save_frame(f, "0") + del f + f = RMF.open_rmf_file_read_only(name) + h2 = IMP.rmf.create_hierarchies(f, m) + self.check_provenance(h2[0]) + +if __name__ == '__main__': + IMP.test.main() From d831ab612b4e4233ca5d22be1e6a7c0897c152c7 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 17 Oct 2017 18:09:05 -0700 Subject: [PATCH 14/19] Destroy provenance information with the Hierarchy. If an atom::Hierarchy has any attached provenance information, this should be destroyed with the Hierarchy when atom::destroy() is called. --- modules/atom/src/Hierarchy.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/atom/src/Hierarchy.cpp b/modules/atom/src/Hierarchy.cpp index d2012832c9..64d75379d0 100644 --- a/modules/atom/src/Hierarchy.cpp +++ b/modules/atom/src/Hierarchy.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -531,6 +532,14 @@ void destroy(Hierarchy d) { destroy_bond(b.get_bond(b.get_number_of_bonds() - 1)); } } + if (Provenanced::get_is_setup(all[i])) { + Provenance prov = Provenanced(all[i]).get_provenance(); + while (prov) { + Provenance previous = prov.get_previous(); + prov.get_model()->remove_particle(prov.get_particle_index()); + prov = previous; + } + } Hierarchy hc(all[i]); while (hc.get_number_of_children() > 0) { hc.remove_child(hc.get_child(hc.get_number_of_children() - 1)); From b1c1ff8388aced07dff2cdbcfe39a8495966ce01 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 17 Oct 2017 18:27:16 -0700 Subject: [PATCH 15/19] Move provenance classes from atom to core. While the provenance decorators are designed to be attached to an atom::Hierarchy, they don't require any functionality from the atom module, so it's more appropriate for them to live in the core module. --- modules/atom/pyext/swig.i-in | 8 --- modules/atom/src/Hierarchy.cpp | 8 +-- modules/atom/src/pdb.cpp | 8 +-- modules/atom/test/test_pdb.py | 9 +-- modules/{atom => core}/include/provenance.h | 30 +++++----- modules/core/pyext/swig.i-in | 8 +++ modules/{atom => core}/src/provenance.cpp | 6 +- .../{atom => core}/test/test_provenance.py | 56 +++++++++---------- modules/rmf/include/atom_links.h | 4 +- modules/rmf/src/atom_links.cpp | 46 +++++++-------- modules/rmf/test/test_provenance.py | 52 ++++++++--------- 11 files changed, 118 insertions(+), 117 deletions(-) rename modules/{atom => core}/include/provenance.h (94%) rename modules/{atom => core}/src/provenance.cpp (97%) rename modules/{atom => core}/test/test_provenance.py (65%) diff --git a/modules/atom/pyext/swig.i-in b/modules/atom/pyext/swig.i-in index a3e2d06cd5..46a4b6f763 100644 --- a/modules/atom/pyext/swig.i-in +++ b/modules/atom/pyext/swig.i-in @@ -37,13 +37,6 @@ IMP_SWIG_DECORATOR(IMP::atom, State, States); IMP_SWIG_DECORATOR(IMP::atom, Copy, Copies); IMP_SWIG_DECORATOR(IMP::atom, SecondaryStructureResidue, SecondaryStructureResidues); IMP_SWIG_DECORATOR(IMP::atom, StructureSource, StructureSources); -IMP_SWIG_DECORATOR(IMP::atom, Provenance, Provenances); -IMP_SWIG_DECORATOR(IMP::atom, StructureProvenance, StructureProvenances); -IMP_SWIG_DECORATOR(IMP::atom, SampleProvenance, SampleProvenances); -IMP_SWIG_DECORATOR(IMP::atom, ClusterProvenance, ClusterProvenances); -IMP_SWIG_DECORATOR(IMP::atom, CombineProvenance, CombineProvenances); -IMP_SWIG_DECORATOR(IMP::atom, FilterProvenance, FilterProvenances); -IMP_SWIG_DECORATOR(IMP::atom, Provenanced, Provenanceds); IMP_SWIG_BASE_OBJECT(IMP::atom, Simulator, Simulators); IMP_SWIG_DIRECTOR(IMP::atom, PDBSelector); IMP_SWIG_OBJECT(IMP::atom, ATOMPDBSelector, ATOMPDBSelectors); @@ -243,7 +236,6 @@ IMP_SWIG_VALUE_INSTANCE(IMP::atom, CHARMMAngle, CHARMMConnection, CHARMMAngles); %include "IMP/atom/Mass.h" %include "IMP/atom/BondedPairFilter.h" %include "IMP/atom/mol2.h" -%include "IMP/atom/provenance.h" %include "IMP/atom/SecondaryStructureResidue.h" %include "IMP/atom/secondary_structure_reader.h" diff --git a/modules/atom/src/Hierarchy.cpp b/modules/atom/src/Hierarchy.cpp index 64d75379d0..460179d8e2 100644 --- a/modules/atom/src/Hierarchy.cpp +++ b/modules/atom/src/Hierarchy.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include @@ -532,10 +532,10 @@ void destroy(Hierarchy d) { destroy_bond(b.get_bond(b.get_number_of_bonds() - 1)); } } - if (Provenanced::get_is_setup(all[i])) { - Provenance prov = Provenanced(all[i]).get_provenance(); + if (core::Provenanced::get_is_setup(all[i])) { + core::Provenance prov = core::Provenanced(all[i]).get_provenance(); while (prov) { - Provenance previous = prov.get_previous(); + core::Provenance previous = prov.get_previous(); prov.get_model()->remove_particle(prov.get_particle_index()); prov = previous; } diff --git a/modules/atom/src/pdb.cpp b/modules/atom/src/pdb.cpp index 345429be95..8ae10f84af 100644 --- a/modules/atom/src/pdb.cpp +++ b/modules/atom/src/pdb.cpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include @@ -252,10 +252,10 @@ Particle* chain_particle(Model* m, char chain_id, std::string filename) { Molecule::setup_particle(p); // Set provenance of this chain - StructureProvenance sp - = StructureProvenance::setup_particle(new Particle(m), + core::StructureProvenance sp + = core::StructureProvenance::setup_particle(new Particle(m), filename, std::string(1, chain_id)); - add_provenance(m, p->get_index(), sp); + core::add_provenance(m, p->get_index(), sp); return p; } diff --git a/modules/atom/test/test_pdb.py b/modules/atom/test/test_pdb.py index 3228098b45..4d1a9eb0df 100644 --- a/modules/atom/test/test_pdb.py +++ b/modules/atom/test/test_pdb.py @@ -3,6 +3,7 @@ import IMP import IMP.test import IMP.atom +import IMP.core class Tests(IMP.test.TestCase): @@ -226,10 +227,10 @@ def test_provenance(self): chains = IMP.atom.get_by_type(mp, IMP.atom.CHAIN_TYPE) self.assertEqual(len(chains), 5) for c in chains: - self.assertTrue(IMP.atom.Provenanced.get_is_setup(c)) - p = IMP.atom.Provenanced(c).get_provenance() - self.assertTrue(IMP.atom.StructureProvenance.get_is_setup(p)) - sp = IMP.atom.StructureProvenance(p) + self.assertTrue(IMP.core.Provenanced.get_is_setup(c)) + p = IMP.core.Provenanced(c).get_provenance() + self.assertTrue(IMP.core.StructureProvenance.get_is_setup(p)) + sp = IMP.core.StructureProvenance(p) self.assertEqual(sp.get_filename(), fname) self.assertEqual(sp.get_chain_id(), IMP.atom.Chain(c).get_id()) diff --git a/modules/atom/include/provenance.h b/modules/core/include/provenance.h similarity index 94% rename from modules/atom/include/provenance.h rename to modules/core/include/provenance.h index 9684ebca9a..ad92100ad2 100644 --- a/modules/atom/include/provenance.h +++ b/modules/core/include/provenance.h @@ -1,14 +1,14 @@ /** - * \file IMP/atom/provenance.h + * \file IMP/core/provenance.h * \brief Classes to track how the model was created. * * Copyright 2007-2017 IMP Inventors. All rights reserved. */ -#ifndef IMPATOM_PROVENANCE_H -#define IMPATOM_PROVENANCE_H +#ifndef IMPCORE_PROVENANCE_H +#define IMPCORE_PROVENANCE_H -#include +#include #include #include @@ -17,7 +17,7 @@ #include #include -IMPATOM_BEGIN_NAMESPACE +IMPCORE_BEGIN_NAMESPACE //! Track how parts of the system were created. /** Particles are linked with this decorator into a directed acyclic graph @@ -27,7 +27,7 @@ IMPATOM_BEGIN_NAMESPACE Typically, part of an IMP::Model (usually an atom::Hierarchy particle) is decorated with Provenanced that points to the root of this graph. */ -class IMPATOMEXPORT Provenance : public Decorator { +class IMPCOREEXPORT Provenance : public Decorator { static void do_setup_particle(Model *m, ParticleIndex pi) { // Use self-index to indicate no previous provenance is set yet m->add_attribute(get_previous_key(), pi, pi); @@ -74,7 +74,7 @@ class IMPATOMEXPORT Provenance : public Decorator { }; //! Track creation of a system fragment from a PDB file. -class IMPATOMEXPORT StructureProvenance : public Provenance { +class IMPCOREEXPORT StructureProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, std::string filename, std::string chain_id) { @@ -129,7 +129,7 @@ class IMPATOMEXPORT StructureProvenance : public Provenance { each frame can be stored, if known and applicable. The rest of the frames are generally stored in a file (e.g. an RMF file). */ -class IMPATOMEXPORT SampleProvenance : public Provenance { +class IMPCOREEXPORT SampleProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, std::string method, int frames) { Provenance::setup_particle(m, pi); @@ -200,7 +200,7 @@ class IMPATOMEXPORT SampleProvenance : public Provenance { essentially identical, differing at most only in the number of frames. The total size of the resulting ensemble is stored here. */ -class IMPATOMEXPORT CombineProvenance : public Provenance { +class IMPCOREEXPORT CombineProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, int runs, int frames) { Provenance::setup_particle(m, pi); @@ -249,7 +249,7 @@ class IMPATOMEXPORT CombineProvenance : public Provenance { resulted from filtering a larger ensemble (the 'previous' provenance) by discarding models with scores above the threshold. */ -class IMPATOMEXPORT FilterProvenance : public Provenance { +class IMPCOREEXPORT FilterProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, double threshold, int frames) { Provenance::setup_particle(m, pi); @@ -299,7 +299,7 @@ class IMPATOMEXPORT FilterProvenance : public Provenance { specified size. The rest of the cluster members are generally stored in a file (e.g. an RMF file). */ -class IMPATOMEXPORT ClusterProvenance : public Provenance { +class IMPCOREEXPORT ClusterProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, int members) { Provenance::setup_particle(m, pi); m->add_attribute(get_members_key(), pi, members); @@ -328,7 +328,7 @@ class IMPATOMEXPORT ClusterProvenance : public Provenance { }; //! Tag part of the system to track how it was created. -class IMPATOMEXPORT Provenanced : public Decorator { +class IMPCOREEXPORT Provenanced : public Decorator { static void do_setup_particle(Model *m, ParticleIndex pi, Provenance p) { m->add_attribute(get_provenance_key(), pi, p.get_particle_index()); @@ -357,9 +357,9 @@ class IMPATOMEXPORT Provenanced : public Decorator { }; //! Add provenance to part of the model. -IMPATOMEXPORT void add_provenance(Model *m, ParticleIndex pi, +IMPCOREEXPORT void add_provenance(Model *m, ParticleIndex pi, Provenance p); -IMPATOM_END_NAMESPACE +IMPCORE_END_NAMESPACE -#endif /* IMPATOM_PROVENANCE_H */ +#endif /* IMPCORE_PROVENANCE_H */ diff --git a/modules/core/pyext/swig.i-in b/modules/core/pyext/swig.i-in index b55fe442b5..af8207ef74 100644 --- a/modules/core/pyext/swig.i-in +++ b/modules/core/pyext/swig.i-in @@ -154,6 +154,13 @@ IMP_SWIG_DIRECTOR(IMP::core, HierarchyVisitor); IMP_SWIG_DECORATOR( IMP::core, Direction, Directions); IMP_SWIG_DECORATOR( IMP::core, DirectionAngle, DirectionAngles); IMP_SWIG_DECORATOR( IMP::core, Surface, Surfaces); +IMP_SWIG_DECORATOR(IMP::core, Provenance, Provenances); +IMP_SWIG_DECORATOR(IMP::core, StructureProvenance, StructureProvenances); +IMP_SWIG_DECORATOR(IMP::core, SampleProvenance, SampleProvenances); +IMP_SWIG_DECORATOR(IMP::core, ClusterProvenance, ClusterProvenances); +IMP_SWIG_DECORATOR(IMP::core, CombineProvenance, CombineProvenances); +IMP_SWIG_DECORATOR(IMP::core, FilterProvenance, FilterProvenances); +IMP_SWIG_DECORATOR(IMP::core, Provenanced, Provenanceds); IMP_SWIG_VALUE( IMP::core, HierarchyCounter, HierarchyCounters); IMP_SWIG_VALUE( IMP::core, HierarchyTraits, HierarchyTraitsList); @@ -348,6 +355,7 @@ void visit_depth_first(Hierarchy d, HierarchyVisitor *f) %include "IMP/core/model_statistics.h" %include "IMP/core/blame.h" %include "IMP/core/MultipleBinormalRestraint.h" +%include "IMP/core/provenance.h" %inline %{ namespace IMP { diff --git a/modules/atom/src/provenance.cpp b/modules/core/src/provenance.cpp similarity index 97% rename from modules/atom/src/provenance.cpp rename to modules/core/src/provenance.cpp index ef44994e54..57656b827c 100644 --- a/modules/atom/src/provenance.cpp +++ b/modules/core/src/provenance.cpp @@ -5,9 +5,9 @@ * Copyright 2007-2017 IMP Inventors. All rights reserved. */ -#include +#include -IMPATOM_BEGIN_NAMESPACE +IMPCORE_BEGIN_NAMESPACE ParticleIndexKey Provenanced::get_provenance_key() { static const ParticleIndexKey provenance("provenance"); @@ -124,4 +124,4 @@ void add_provenance(Model *m, ParticleIndex pi, Provenance p) { } } -IMPATOM_END_NAMESPACE +IMPCORE_END_NAMESPACE diff --git a/modules/atom/test/test_provenance.py b/modules/core/test/test_provenance.py similarity index 65% rename from modules/atom/test/test_provenance.py rename to modules/core/test/test_provenance.py index 60f6ea0dda..71f574ef85 100644 --- a/modules/atom/test/test_provenance.py +++ b/modules/core/test/test_provenance.py @@ -1,36 +1,36 @@ from __future__ import print_function import IMP import IMP.test -import IMP.atom +import IMP.core class Tests(IMP.test.TestCase): def test_provenance(self): """Test Provenance decorator""" m = IMP.Model() - p = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) - self.assertTrue(IMP.atom.Provenance.get_is_setup(p)) + p = IMP.core.Provenance.setup_particle(m, IMP.Particle(m)) + self.assertTrue(IMP.core.Provenance.get_is_setup(p)) self.assertFalse(p.get_previous()) - p2 = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) + p2 = IMP.core.Provenance.setup_particle(m, IMP.Particle(m)) p.set_previous(p2) self.assertEqual(p.get_previous(), p2) def test_structure_provenance(self): """Test StructureProvenance decorator""" m = IMP.Model() - p = IMP.atom.StructureProvenance.setup_particle(m, IMP.Particle(m), + p = IMP.core.StructureProvenance.setup_particle(m, IMP.Particle(m), "testfile", "testchain") - self.assertTrue(IMP.atom.StructureProvenance.get_is_setup(p)) + self.assertTrue(IMP.core.StructureProvenance.get_is_setup(p)) self.assertEqual(p.get_filename(), "testfile") self.assertEqual(p.get_chain_id(), "testchain") def test_sample_provenance(self): """Test SampleProvenance decorator""" m = IMP.Model() - p = IMP.atom.SampleProvenance.setup_particle(m, IMP.Particle(m), + p = IMP.core.SampleProvenance.setup_particle(m, IMP.Particle(m), "Monte Carlo", 100) - self.assertTrue(IMP.atom.SampleProvenance.get_is_setup(p)) + self.assertTrue(IMP.core.SampleProvenance.get_is_setup(p)) self.assertEqual(p.get_method(), "Monte Carlo") p.set_method("Molecular Dynamics") self.assertEqual(p.get_method(), "Molecular Dynamics") @@ -46,8 +46,8 @@ def test_sample_provenance(self): def test_combine_provenance(self): """Test CombineProvenance decorator""" m = IMP.Model() - p = IMP.atom.CombineProvenance.setup_particle(m, IMP.Particle(m), 5, 42) - self.assertTrue(IMP.atom.CombineProvenance.get_is_setup(p)) + p = IMP.core.CombineProvenance.setup_particle(m, IMP.Particle(m), 5, 42) + self.assertTrue(IMP.core.CombineProvenance.get_is_setup(p)) self.assertEqual(p.get_number_of_runs(), 5) p.set_number_of_runs(7) self.assertEqual(p.get_number_of_runs(), 7) @@ -58,9 +58,9 @@ def test_combine_provenance(self): def test_filter_provenance(self): """Test FilterProvenance decorator""" m = IMP.Model() - p = IMP.atom.FilterProvenance.setup_particle(m, IMP.Particle(m), 100.5, + p = IMP.core.FilterProvenance.setup_particle(m, IMP.Particle(m), 100.5, 42) - self.assertTrue(IMP.atom.FilterProvenance.get_is_setup(p)) + self.assertTrue(IMP.core.FilterProvenance.get_is_setup(p)) self.assertAlmostEqual(p.get_threshold(), 100.5, delta=0.01) p.set_threshold(76.0) self.assertAlmostEqual(p.get_threshold(), 76.0, delta=0.01) @@ -71,8 +71,8 @@ def test_filter_provenance(self): def test_cluster_provenance(self): """Test ClusterProvenance decorator""" m = IMP.Model() - p = IMP.atom.ClusterProvenance.setup_particle(m, IMP.Particle(m), 10) - self.assertTrue(IMP.atom.ClusterProvenance.get_is_setup(p)) + p = IMP.core.ClusterProvenance.setup_particle(m, IMP.Particle(m), 10) + self.assertTrue(IMP.core.ClusterProvenance.get_is_setup(p)) self.assertEqual(p.get_number_of_members(), 10) p.set_number_of_members(42) self.assertEqual(p.get_number_of_members(), 42) @@ -80,32 +80,32 @@ def test_cluster_provenance(self): def test_provenanced(self): """Test Provenanced decorator""" m = IMP.Model() - p = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) - pd = IMP.atom.Provenanced.setup_particle(m, IMP.Particle(m), p) + p = IMP.core.Provenance.setup_particle(m, IMP.Particle(m)) + pd = IMP.core.Provenanced.setup_particle(m, IMP.Particle(m), p) self.assertEqual(pd.get_provenance(), p) pd.set_provenance(p) self.assertEqual(pd.get_provenance(), p) - self.assertTrue(IMP.atom.Provenanced.get_is_setup(pd)) - self.assertTrue(IMP.atom.Provenance.get_is_setup(p)) - self.assertFalse(IMP.atom.Provenanced.get_is_setup(p)) - self.assertFalse(IMP.atom.Provenance.get_is_setup(pd)) + self.assertTrue(IMP.core.Provenanced.get_is_setup(pd)) + self.assertTrue(IMP.core.Provenance.get_is_setup(p)) + self.assertFalse(IMP.core.Provenanced.get_is_setup(p)) + self.assertFalse(IMP.core.Provenance.get_is_setup(pd)) def test_add_provenance(self): """Test add_provenance()""" m = IMP.Model() - prov1 = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) - prov2 = IMP.atom.Provenance.setup_particle(m, IMP.Particle(m)) + prov1 = IMP.core.Provenance.setup_particle(m, IMP.Particle(m)) + prov2 = IMP.core.Provenance.setup_particle(m, IMP.Particle(m)) p = IMP.Particle(m) - self.assertFalse(IMP.atom.Provenanced.get_is_setup(p)) + self.assertFalse(IMP.core.Provenanced.get_is_setup(p)) - IMP.atom.add_provenance(m, p, prov1) - self.assertTrue(IMP.atom.Provenanced.get_is_setup(p)) - pd = IMP.atom.Provenanced(p) + IMP.core.add_provenance(m, p, prov1) + self.assertTrue(IMP.core.Provenanced.get_is_setup(p)) + pd = IMP.core.Provenanced(p) self.assertTrue(pd.get_provenance(), prov1) - IMP.atom.add_provenance(m, p, prov2) - self.assertTrue(IMP.atom.Provenanced.get_is_setup(p)) + IMP.core.add_provenance(m, p, prov2) + self.assertTrue(IMP.core.Provenanced.get_is_setup(p)) self.assertTrue(pd.get_provenance(), prov2) self.assertTrue(pd.get_provenance().get_previous(), prov1) diff --git a/modules/rmf/include/atom_links.h b/modules/rmf/include/atom_links.h index c2262e4498..878dba3298 100644 --- a/modules/rmf/include/atom_links.h +++ b/modules/rmf/include/atom_links.h @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include @@ -85,7 +85,7 @@ class IMPRMFEXPORT HierarchyLoadLink : public SimpleLoadLink { ParticleIndex cur); // Make *Provenance IMP particle corresponding to that in the RMF - atom::Provenance create_one_provenance(Model *m, RMF::NodeConstHandle node); + core::Provenance create_one_provenance(Model *m, RMF::NodeConstHandle node); virtual void do_load_one(RMF::NodeConstHandle nh, Particle *o) IMP_FINAL IMP_OVERRIDE; diff --git a/modules/rmf/src/atom_links.cpp b/modules/rmf/src/atom_links.cpp index 0f92fb0dab..bf40dcfcf4 100644 --- a/modules/rmf/src/atom_links.cpp +++ b/modules/rmf/src/atom_links.cpp @@ -171,45 +171,45 @@ void HierarchyLoadLink::create_recursive(Model *m, // Make *Provenance IMP particles corresponding to those in the RMF void HierarchyLoadLink::create_provenance(Model *m, RMF::NodeConstHandle node, ParticleIndex cur) { - atom::Provenance prov = create_one_provenance(m, node); - atom::Provenanced provd = atom::Provenanced::setup_particle(m, cur, prov); + core::Provenance prov = create_one_provenance(m, node); + core::Provenanced provd = core::Provenanced::setup_particle(m, cur, prov); while ((node = get_previous_rmf_provenance(node)) != RMF::NodeConstHandle()) { - atom::Provenance thisprov = create_one_provenance(m, node); + core::Provenance thisprov = create_one_provenance(m, node); prov.set_previous(thisprov); prov = thisprov; } } -atom::Provenance HierarchyLoadLink::create_one_provenance(Model *m, +core::Provenance HierarchyLoadLink::create_one_provenance(Model *m, RMF::NodeConstHandle node) { if (strucpf_.get_is(node)) { RMF::decorator::StructureProvenanceConst rp = strucpf_.get(node); ParticleIndex ip = m->add_particle(node.get_name()); - return atom::StructureProvenance::setup_particle(m, ip, rp.get_filename(), + return core::StructureProvenance::setup_particle(m, ip, rp.get_filename(), rp.get_chain()); } else if (samppf_.get_is(node)) { RMF::decorator::SampleProvenanceConst rp = samppf_.get(node); ParticleIndex ip = m->add_particle(node.get_name()); - atom::SampleProvenance sp = - atom::SampleProvenance::setup_particle(m, ip, rp.get_method(), + core::SampleProvenance sp = + core::SampleProvenance::setup_particle(m, ip, rp.get_method(), rp.get_frames()); sp.set_number_of_iterations(rp.get_iterations()); return sp; } else if (combpf_.get_is(node)) { RMF::decorator::CombineProvenanceConst rp = combpf_.get(node); ParticleIndex ip = m->add_particle(node.get_name()); - return atom::CombineProvenance::setup_particle(m, ip, rp.get_runs(), + return core::CombineProvenance::setup_particle(m, ip, rp.get_runs(), rp.get_frames()); } else if (filtpf_.get_is(node)) { RMF::decorator::FilterProvenanceConst rp = filtpf_.get(node); ParticleIndex ip = m->add_particle(node.get_name()); - return atom::FilterProvenance::setup_particle(m, ip, rp.get_threshold(), + return core::FilterProvenance::setup_particle(m, ip, rp.get_threshold(), rp.get_frames()); } else if (clustpf_.get_is(node)) { RMF::decorator::ClusterProvenanceConst rp = clustpf_.get(node); ParticleIndex ip = m->add_particle(node.get_name()); - return atom::ClusterProvenance::setup_particle(m, ip, rp.get_members()); + return core::ClusterProvenance::setup_particle(m, ip, rp.get_members()); } else { IMP_THROW("Unhandled provenance type " << node, IOException); } @@ -420,7 +420,7 @@ void HierarchySaveLink::add_recursive(Model *m, ParticleIndex root, } } - if (atom::Provenanced::get_is_setup(m, p)) { + if (core::Provenanced::get_is_setup(m, p)) { add_provenance(m, p, cur); } @@ -438,40 +438,40 @@ void HierarchySaveLink::add_recursive(Model *m, ParticleIndex root, // Make RMF PROVENANCE nodes corresponding to those in IMP void HierarchySaveLink::add_provenance(Model *m, ParticleIndex p, RMF::NodeHandle cur) { - atom::Provenanced provd(m, p); - for (atom::Provenance prov = provd.get_provenance(); prov; + core::Provenanced provd(m, p); + for (core::Provenance prov = provd.get_provenance(); prov; prov = prov.get_previous()) { - if (atom::StructureProvenance::get_is_setup(prov)) { - atom::StructureProvenance ip(prov); + if (core::StructureProvenance::get_is_setup(prov)) { + core::StructureProvenance ip(prov); cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), RMF::PROVENANCE); RMF::decorator::StructureProvenance rp = strucpf_.get(cur); rp.set_filename(ip.get_filename()); rp.set_chain(ip.get_chain_id()); - } else if (atom::SampleProvenance::get_is_setup(prov)) { - atom::SampleProvenance ip(prov); + } else if (core::SampleProvenance::get_is_setup(prov)) { + core::SampleProvenance ip(prov); cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), RMF::PROVENANCE); RMF::decorator::SampleProvenance rp = samppf_.get(cur); rp.set_method(ip.get_method()); rp.set_frames(ip.get_number_of_frames()); rp.set_iterations(ip.get_number_of_iterations()); - } else if (atom::CombineProvenance::get_is_setup(prov)) { - atom::CombineProvenance ip(prov); + } else if (core::CombineProvenance::get_is_setup(prov)) { + core::CombineProvenance ip(prov); cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), RMF::PROVENANCE); RMF::decorator::CombineProvenance rp = combpf_.get(cur); rp.set_runs(ip.get_number_of_runs()); rp.set_frames(ip.get_number_of_frames()); - } else if (atom::FilterProvenance::get_is_setup(prov)) { - atom::FilterProvenance ip(prov); + } else if (core::FilterProvenance::get_is_setup(prov)) { + core::FilterProvenance ip(prov); cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), RMF::PROVENANCE); RMF::decorator::FilterProvenance rp = filtpf_.get(cur); rp.set_threshold(ip.get_threshold()); rp.set_frames(ip.get_number_of_frames()); - } else if (atom::ClusterProvenance::get_is_setup(prov)) { - atom::ClusterProvenance ip(prov); + } else if (core::ClusterProvenance::get_is_setup(prov)) { + core::ClusterProvenance ip(prov); cur = cur.add_child(m->get_particle_name(prov.get_particle_index()), RMF::PROVENANCE); RMF::decorator::ClusterProvenance rp = clustpf_.get(cur); diff --git a/modules/rmf/test/test_provenance.py b/modules/rmf/test/test_provenance.py index 892d195cfa..69af4069cc 100644 --- a/modules/rmf/test/test_provenance.py +++ b/modules/rmf/test/test_provenance.py @@ -9,36 +9,36 @@ class Tests(IMP.test.TestCase): def add_provenance(self, h): m = h.get_model() - struc = IMP.atom.StructureProvenance.setup_particle( + struc = IMP.core.StructureProvenance.setup_particle( m, IMP.Particle(m), "testfile", "testchain") struc.set_name("structure provenance") - IMP.atom.add_provenance(m, h, struc) + IMP.core.add_provenance(m, h, struc) - samp = IMP.atom.SampleProvenance.setup_particle( + samp = IMP.core.SampleProvenance.setup_particle( m, IMP.Particle(m), "Monte Carlo", 100) samp.set_number_of_iterations(42) - IMP.atom.add_provenance(m, h, samp) + IMP.core.add_provenance(m, h, samp) - comb = IMP.atom.CombineProvenance.setup_particle( + comb = IMP.core.CombineProvenance.setup_particle( m, IMP.Particle(m), 4, 27) - IMP.atom.add_provenance(m, h, comb) + IMP.core.add_provenance(m, h, comb) - filt = IMP.atom.FilterProvenance.setup_particle( + filt = IMP.core.FilterProvenance.setup_particle( m, IMP.Particle(m), 100.5, 39) - IMP.atom.add_provenance(m, h, filt) + IMP.core.add_provenance(m, h, filt) - clus = IMP.atom.ClusterProvenance.setup_particle(m, IMP.Particle(m), 10) - IMP.atom.add_provenance(m, h, clus) + clus = IMP.core.ClusterProvenance.setup_particle(m, IMP.Particle(m), 10) + IMP.core.add_provenance(m, h, clus) def check_provenance(self, h): m = h.get_model() # Test IMP-added chain provenance chain, = IMP.atom.get_by_type(h, IMP.atom.CHAIN_TYPE) - self.assertTrue(IMP.atom.Provenanced.get_is_setup(m, chain)) - prov = IMP.atom.Provenanced(m, chain).get_provenance() - self.assertTrue(IMP.atom.StructureProvenance.get_is_setup(m, prov)) - struc = IMP.atom.StructureProvenance(m, prov) + self.assertTrue(IMP.core.Provenanced.get_is_setup(m, chain)) + prov = IMP.core.Provenanced(m, chain).get_provenance() + self.assertTrue(IMP.core.StructureProvenance.get_is_setup(m, prov)) + struc = IMP.core.StructureProvenance(m, prov) self.assertEqual(struc.get_chain_id(), 'A') # Should be no more chain provenance @@ -46,35 +46,35 @@ def check_provenance(self, h): self.assertFalse(prov) # Check the provenance we added at the top level - self.assertTrue(IMP.atom.Provenanced.get_is_setup(m, h)) - prov = IMP.atom.Provenanced(m, h).get_provenance() + self.assertTrue(IMP.core.Provenanced.get_is_setup(m, h)) + prov = IMP.core.Provenanced(m, h).get_provenance() - self.assertTrue(IMP.atom.ClusterProvenance.get_is_setup(m, prov)) - clus = IMP.atom.ClusterProvenance(m, prov) + self.assertTrue(IMP.core.ClusterProvenance.get_is_setup(m, prov)) + clus = IMP.core.ClusterProvenance(m, prov) self.assertEqual(clus.get_number_of_members(), 10) prov = prov.get_previous() - self.assertTrue(IMP.atom.FilterProvenance.get_is_setup(m, prov)) - filt = IMP.atom.FilterProvenance(m, prov) + self.assertTrue(IMP.core.FilterProvenance.get_is_setup(m, prov)) + filt = IMP.core.FilterProvenance(m, prov) self.assertAlmostEqual(filt.get_threshold(), 100.5, delta=1e-4) self.assertEqual(filt.get_number_of_frames(), 39) prov = prov.get_previous() - self.assertTrue(IMP.atom.CombineProvenance.get_is_setup(m, prov)) - comb = IMP.atom.CombineProvenance(m, prov) + self.assertTrue(IMP.core.CombineProvenance.get_is_setup(m, prov)) + comb = IMP.core.CombineProvenance(m, prov) self.assertEqual(comb.get_number_of_runs(), 4) self.assertEqual(comb.get_number_of_frames(), 27) prov = prov.get_previous() - self.assertTrue(IMP.atom.SampleProvenance.get_is_setup(m, prov)) - samp = IMP.atom.SampleProvenance(m, prov) + self.assertTrue(IMP.core.SampleProvenance.get_is_setup(m, prov)) + samp = IMP.core.SampleProvenance(m, prov) self.assertEqual(samp.get_method(), "Monte Carlo") self.assertEqual(samp.get_number_of_frames(), 100) self.assertEqual(samp.get_number_of_iterations(), 42) prov = prov.get_previous() - self.assertTrue(IMP.atom.StructureProvenance.get_is_setup(m, prov)) - struc = IMP.atom.StructureProvenance(m, prov) + self.assertTrue(IMP.core.StructureProvenance.get_is_setup(m, prov)) + struc = IMP.core.StructureProvenance(m, prov) self.assertEqual(struc.get_filename(), "testfile") self.assertEqual(struc.get_chain_id(), "testchain") self.assertEqual(struc.get_name(), "structure provenance") From 7400df9e29fbe7083a427c8ff033f826fa765b1d Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 18 Oct 2017 09:46:29 -0700 Subject: [PATCH 16/19] Simplify setup of SampleProvenance decorator. - Allow setting up all attributes. - Check method name when provided at setup time. --- modules/core/include/provenance.h | 19 +++++++++++++------ modules/core/test/test_provenance.py | 7 +++++-- modules/rmf/src/atom_links.cpp | 7 ++----- modules/rmf/test/test_provenance.py | 3 +-- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/modules/core/include/provenance.h b/modules/core/include/provenance.h index ad92100ad2..fa020e7ed5 100644 --- a/modules/core/include/provenance.h +++ b/modules/core/include/provenance.h @@ -131,11 +131,13 @@ class IMPCOREEXPORT StructureProvenance : public Provenance { */ class IMPCOREEXPORT SampleProvenance : public Provenance { static void do_setup_particle(Model *m, ParticleIndex pi, - std::string method, int frames) { + std::string method, int frames, + int iterations) { + validate_method(method); Provenance::setup_particle(m, pi); m->add_attribute(get_method_key(), pi, method); m->add_attribute(get_frames_key(), pi, frames); - m->add_attribute(get_iterations_key(), pi, 1); + m->add_attribute(get_iterations_key(), pi, iterations); } static StringKey get_method_key(); @@ -144,6 +146,12 @@ class IMPCOREEXPORT SampleProvenance : public Provenance { static std::set& get_allowed_methods(); + static void validate_method(std::string method) { + IMP_USAGE_CHECK(get_allowed_methods().find(method) + != get_allowed_methods().end(), + "Invalid sampling method"); + } + public: static bool get_is_setup(Model *m, ParticleIndex pi) { return m->get_has_attribute(get_method_key(), pi) @@ -153,9 +161,7 @@ class IMPCOREEXPORT SampleProvenance : public Provenance { //! Set the sampling method void set_method(std::string method) const { - IMP_USAGE_CHECK(get_allowed_methods().find(method) - != get_allowed_methods().end(), - "Invalid sampling method"); + validate_method(method); return get_model()->set_attribute(get_method_key(), get_particle_index(), method); } @@ -189,7 +195,8 @@ class IMPCOREEXPORT SampleProvenance : public Provenance { } IMP_DECORATOR_METHODS(SampleProvenance, Provenance); - IMP_DECORATOR_SETUP_2(SampleProvenance, std::string, method, int, frames); + IMP_DECORATOR_SETUP_3(SampleProvenance, std::string, method, int, frames, + int, iterations); }; //! Track creation of a system fragment by combination. diff --git a/modules/core/test/test_provenance.py b/modules/core/test/test_provenance.py index 71f574ef85..9125f77ad6 100644 --- a/modules/core/test/test_provenance.py +++ b/modules/core/test/test_provenance.py @@ -29,17 +29,20 @@ def test_sample_provenance(self): """Test SampleProvenance decorator""" m = IMP.Model() p = IMP.core.SampleProvenance.setup_particle(m, IMP.Particle(m), - "Monte Carlo", 100) + "Monte Carlo", 100, 5) self.assertTrue(IMP.core.SampleProvenance.get_is_setup(p)) self.assertEqual(p.get_method(), "Monte Carlo") p.set_method("Molecular Dynamics") self.assertEqual(p.get_method(), "Molecular Dynamics") if IMP.get_check_level() == IMP.USAGE_AND_INTERNAL: self.assertRaises(IMP.UsageError, p.set_method, "Garbage") + self.assertRaises(IMP.UsageError, + IMP.core.SampleProvenance.setup_particle, m, IMP.Particle(m), + "Garbage", 100) self.assertEqual(p.get_number_of_frames(), 100) p.set_number_of_frames(200) self.assertEqual(p.get_number_of_frames(), 200) - self.assertEqual(p.get_number_of_iterations(), 1) + self.assertEqual(p.get_number_of_iterations(), 5) p.set_number_of_iterations(42) self.assertEqual(p.get_number_of_iterations(), 42) diff --git a/modules/rmf/src/atom_links.cpp b/modules/rmf/src/atom_links.cpp index bf40dcfcf4..0935202880 100644 --- a/modules/rmf/src/atom_links.cpp +++ b/modules/rmf/src/atom_links.cpp @@ -191,11 +191,8 @@ core::Provenance HierarchyLoadLink::create_one_provenance(Model *m, } else if (samppf_.get_is(node)) { RMF::decorator::SampleProvenanceConst rp = samppf_.get(node); ParticleIndex ip = m->add_particle(node.get_name()); - core::SampleProvenance sp = - core::SampleProvenance::setup_particle(m, ip, rp.get_method(), - rp.get_frames()); - sp.set_number_of_iterations(rp.get_iterations()); - return sp; + return core::SampleProvenance::setup_particle(m, ip, rp.get_method(), + rp.get_frames(), rp.get_iterations()); } else if (combpf_.get_is(node)) { RMF::decorator::CombineProvenanceConst rp = combpf_.get(node); ParticleIndex ip = m->add_particle(node.get_name()); diff --git a/modules/rmf/test/test_provenance.py b/modules/rmf/test/test_provenance.py index 69af4069cc..ae44985b98 100644 --- a/modules/rmf/test/test_provenance.py +++ b/modules/rmf/test/test_provenance.py @@ -15,8 +15,7 @@ def add_provenance(self, h): IMP.core.add_provenance(m, h, struc) samp = IMP.core.SampleProvenance.setup_particle( - m, IMP.Particle(m), "Monte Carlo", 100) - samp.set_number_of_iterations(42) + m, IMP.Particle(m), "Monte Carlo", 100, 42) IMP.core.add_provenance(m, h, samp) comb = IMP.core.CombineProvenance.setup_particle( From 50383dcb608110ed93a37ba56aeba13fd9d8167e Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 18 Oct 2017 11:00:54 -0700 Subject: [PATCH 17/19] Note that destroy() now removes provenance info. --- modules/atom/include/Hierarchy.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/atom/include/Hierarchy.h b/modules/atom/include/Hierarchy.h index ef0fc918c3..89a503ee32 100644 --- a/modules/atom/include/Hierarchy.h +++ b/modules/atom/include/Hierarchy.h @@ -470,7 +470,8 @@ IMPATOMEXPORT Hierarchy create_clone_one(Hierarchy d); /** All bonds connecting to these atoms are destroyed as are hierarchy links in the Hierarchy and the particles are removed from the Model. If this particle has a parent, it is - removed from the parent. + removed from the parent. Any provenance information for this + Hierarchy is also removed. \relates Hierarchy */ IMPATOMEXPORT void destroy(Hierarchy d); From 64300c4e49ec564ff5cc403025c0d55d218eb06a Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 18 Oct 2017 11:04:41 -0700 Subject: [PATCH 18/19] Add a function to clone provenance information. --- modules/core/include/provenance.h | 33 ++++++++++++++ modules/core/src/provenance.cpp | 45 +++++++++++++++++++ modules/core/test/test_provenance.py | 66 ++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) diff --git a/modules/core/include/provenance.h b/modules/core/include/provenance.h index fa020e7ed5..3de4bc092d 100644 --- a/modules/core/include/provenance.h +++ b/modules/core/include/provenance.h @@ -84,6 +84,11 @@ class IMPCOREEXPORT StructureProvenance : public Provenance { m->add_attribute(get_chain_key(), pi, chain_id); } + static void do_setup_particle(Model *m, ParticleIndex pi, + StructureProvenance o) { + do_setup_particle(m, pi, o.get_filename(), o.get_chain_id()); + } + static StringKey get_filename_key(); static StringKey get_chain_key(); @@ -119,6 +124,7 @@ class IMPCOREEXPORT StructureProvenance : public Provenance { IMP_DECORATOR_METHODS(StructureProvenance, Provenance); IMP_DECORATOR_SETUP_2(StructureProvenance, std::string, filename, std::string, chain_id); + IMP_DECORATOR_SETUP_1(StructureProvenance, StructureProvenance, o); }; //! Track creation of a system fragment from sampling. @@ -140,6 +146,12 @@ class IMPCOREEXPORT SampleProvenance : public Provenance { m->add_attribute(get_iterations_key(), pi, iterations); } + static void do_setup_particle(Model *m, ParticleIndex pi, + SampleProvenance o) { + do_setup_particle(m, pi, o.get_method(), o.get_number_of_frames(), + o.get_number_of_iterations()); + } + static StringKey get_method_key(); static IntKey get_frames_key(); static IntKey get_iterations_key(); @@ -197,6 +209,7 @@ class IMPCOREEXPORT SampleProvenance : public Provenance { IMP_DECORATOR_METHODS(SampleProvenance, Provenance); IMP_DECORATOR_SETUP_3(SampleProvenance, std::string, method, int, frames, int, iterations); + IMP_DECORATOR_SETUP_1(SampleProvenance, SampleProvenance, o); }; //! Track creation of a system fragment by combination. @@ -215,6 +228,11 @@ class IMPCOREEXPORT CombineProvenance : public Provenance { m->add_attribute(get_frames_key(), pi, frames); } + static void do_setup_particle(Model *m, ParticleIndex pi, + CombineProvenance o) { + do_setup_particle(m, pi, o.get_number_of_runs(), o.get_number_of_frames()); + } + static IntKey get_runs_key(); static IntKey get_frames_key(); @@ -248,6 +266,7 @@ class IMPCOREEXPORT CombineProvenance : public Provenance { IMP_DECORATOR_METHODS(CombineProvenance, Provenance); IMP_DECORATOR_SETUP_2(CombineProvenance, int, runs, int, frames); + IMP_DECORATOR_SETUP_1(CombineProvenance, CombineProvenance, o); }; //! Track creation of a system fragment by filtering. @@ -263,6 +282,10 @@ class IMPCOREEXPORT FilterProvenance : public Provenance { m->add_attribute(get_threshold_key(), pi, threshold); m->add_attribute(get_frames_key(), pi, frames); } + static void do_setup_particle(Model *m, ParticleIndex pi, + FilterProvenance o) { + do_setup_particle(m, pi, o.get_threshold(), o.get_number_of_frames()); + } static FloatKey get_threshold_key(); static IntKey get_frames_key(); @@ -298,6 +321,7 @@ class IMPCOREEXPORT FilterProvenance : public Provenance { IMP_DECORATOR_METHODS(FilterProvenance, Provenance); IMP_DECORATOR_SETUP_2(FilterProvenance, double, threshold, int, frames); + IMP_DECORATOR_SETUP_1(FilterProvenance, FilterProvenance, o); }; //! Track creation of a system fragment from clustering. @@ -312,6 +336,11 @@ class IMPCOREEXPORT ClusterProvenance : public Provenance { m->add_attribute(get_members_key(), pi, members); } + static void do_setup_particle(Model *m, ParticleIndex pi, + ClusterProvenance o) { + do_setup_particle(m, pi, o.get_number_of_members()); + } + static IntKey get_members_key(); public: @@ -332,6 +361,7 @@ class IMPCOREEXPORT ClusterProvenance : public Provenance { IMP_DECORATOR_METHODS(ClusterProvenance, Provenance); IMP_DECORATOR_SETUP_1(ClusterProvenance, int, members); + IMP_DECORATOR_SETUP_1(ClusterProvenance, ClusterProvenance, o); }; //! Tag part of the system to track how it was created. @@ -367,6 +397,9 @@ class IMPCOREEXPORT Provenanced : public Decorator { IMPCOREEXPORT void add_provenance(Model *m, ParticleIndex pi, Provenance p); +//! Clone provenance (including previous provenance) +IMPCOREEXPORT Provenance create_clone(Provenance p); + IMPCORE_END_NAMESPACE #endif /* IMPCORE_PROVENANCE_H */ diff --git a/modules/core/src/provenance.cpp b/modules/core/src/provenance.cpp index 57656b827c..80ad972eb9 100644 --- a/modules/core/src/provenance.cpp +++ b/modules/core/src/provenance.cpp @@ -9,6 +9,35 @@ IMPCORE_BEGIN_NAMESPACE +namespace { + +Provenance clone_one(Provenance prov) { + Particle *p = new IMP::Particle(prov.get_model()); + p->set_name(prov->get_name()); + + if (StructureProvenance::get_is_setup(prov.get_particle())) { + StructureProvenance::setup_particle(p, + StructureProvenance(prov.get_particle())); + } else if (SampleProvenance::get_is_setup(prov.get_particle())) { + SampleProvenance::setup_particle(p, + SampleProvenance(prov.get_particle())); + } else if (CombineProvenance::get_is_setup(prov.get_particle())) { + CombineProvenance::setup_particle(p, + CombineProvenance(prov.get_particle())); + } else if (FilterProvenance::get_is_setup(prov.get_particle())) { + FilterProvenance::setup_particle(p, + FilterProvenance(prov.get_particle())); + } else if (ClusterProvenance::get_is_setup(prov.get_particle())) { + ClusterProvenance::setup_particle(p, + ClusterProvenance(prov.get_particle())); + } else { + IMP_THROW("Unhandled provenance", IOException); + } + return Provenance(p); +} + +} // anonymous namespace + ParticleIndexKey Provenanced::get_provenance_key() { static const ParticleIndexKey provenance("provenance"); return provenance; @@ -124,4 +153,20 @@ void add_provenance(Model *m, ParticleIndex pi, Provenance p) { } } +Provenance create_clone(Provenance prov) { + Provenance root = clone_one(prov); + + Provenance newprov = root; + while (prov) { + Provenance previous = prov.get_previous(); + if (previous) { + Provenance newprevious = clone_one(previous); + newprov.set_previous(newprevious); + newprov = newprevious; + } + prov = previous; + } + return root; +} + IMPCORE_END_NAMESPACE diff --git a/modules/core/test/test_provenance.py b/modules/core/test/test_provenance.py index 9125f77ad6..2c0aa10f60 100644 --- a/modules/core/test/test_provenance.py +++ b/modules/core/test/test_provenance.py @@ -112,5 +112,71 @@ def test_add_provenance(self): self.assertTrue(pd.get_provenance(), prov2) self.assertTrue(pd.get_provenance().get_previous(), prov1) + def add_provenance(self, m): + struc = IMP.core.StructureProvenance.setup_particle( + m, IMP.Particle(m), "testfile", "testchain") + + samp = IMP.core.SampleProvenance.setup_particle( + m, IMP.Particle(m), "Monte Carlo", 100, 42) + samp.set_previous(struc) + + comb = IMP.core.CombineProvenance.setup_particle( + m, IMP.Particle(m), 4, 27) + comb.set_previous(samp) + + filt = IMP.core.FilterProvenance.setup_particle( + m, IMP.Particle(m), 100.5, 39) + filt.set_previous(comb) + + clus = IMP.core.ClusterProvenance.setup_particle(m, IMP.Particle(m), 10) + clus.set_previous(filt) + return clus + + def check_provenance(self, prov): + m = prov.get_model() + self.assertTrue(IMP.core.ClusterProvenance.get_is_setup(m, prov)) + clus = IMP.core.ClusterProvenance(m, prov) + self.assertEqual(clus.get_number_of_members(), 10) + + prov = prov.get_previous() + self.assertTrue(IMP.core.FilterProvenance.get_is_setup(m, prov)) + filt = IMP.core.FilterProvenance(m, prov) + self.assertAlmostEqual(filt.get_threshold(), 100.5, delta=1e-4) + self.assertEqual(filt.get_number_of_frames(), 39) + + prov = prov.get_previous() + self.assertTrue(IMP.core.CombineProvenance.get_is_setup(m, prov)) + comb = IMP.core.CombineProvenance(m, prov) + self.assertEqual(comb.get_number_of_runs(), 4) + self.assertEqual(comb.get_number_of_frames(), 27) + + prov = prov.get_previous() + self.assertTrue(IMP.core.SampleProvenance.get_is_setup(m, prov)) + samp = IMP.core.SampleProvenance(m, prov) + self.assertEqual(samp.get_method(), "Monte Carlo") + self.assertEqual(samp.get_number_of_frames(), 100) + self.assertEqual(samp.get_number_of_iterations(), 42) + + prov = prov.get_previous() + self.assertTrue(IMP.core.StructureProvenance.get_is_setup(m, prov)) + struc = IMP.core.StructureProvenance(m, prov) + self.assertEqual(struc.get_filename(), "testfile") + self.assertEqual(struc.get_chain_id(), "testchain") + + # Should be no more provenance + prov = prov.get_previous() + self.assertFalse(prov) + + def test_clone(self): + """Test create_clone""" + m = IMP.Model() + prov = self.add_provenance(m) + self.check_provenance(prov) + self.assertEqual(len(m.get_particle_indexes()), 5) + newprov = IMP.core.create_clone(prov) + self.assertEqual(len(m.get_particle_indexes()), 10) + self.check_provenance(newprov) + + if __name__ == '__main__': IMP.test.main() From 992a5830734f5dd567787df2bda67a9e2ddbb401 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 18 Oct 2017 11:05:24 -0700 Subject: [PATCH 19/19] Copy provenance info when cloning a Hierarchy. If any part of the Hierarchy has any attached provenance information, also clone that and attach it to the cloned Hierarchy in the same place. --- modules/atom/include/Hierarchy.h | 4 ++-- modules/atom/src/Hierarchy.cpp | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/atom/include/Hierarchy.h b/modules/atom/include/Hierarchy.h index 89a503ee32..2b7cf34841 100644 --- a/modules/atom/include/Hierarchy.h +++ b/modules/atom/include/Hierarchy.h @@ -451,7 +451,7 @@ IMPATOMEXPORT bool get_is_heterogen(Hierarchy h); //! Clone the Hierarchy /** This method copies the Bond, Bonded, Atom, - Residue, and Domain data and the particle name to the + Residue, Domain, and provenance data and the particle name to the new copies in addition to the Hierarchy relationships. \relates Hierarchy @@ -460,7 +460,7 @@ IMPATOMEXPORT Hierarchy create_clone(Hierarchy d); //! Clone the node in the Hierarchy /** This method copies the Atom, - Residue, Chain and Domain data and the particle name. + Residue, Chain, Domain, and provenance data and the particle name. \relates Hierarchy */ diff --git a/modules/atom/src/Hierarchy.cpp b/modules/atom/src/Hierarchy.cpp index 460179d8e2..41b8cecd0c 100644 --- a/modules/atom/src/Hierarchy.cpp +++ b/modules/atom/src/Hierarchy.cpp @@ -438,6 +438,11 @@ Hierarchy clone_internal(Hierarchy d, if (Representation::get_is_setup(d.get_particle())) { nd = Representation::setup_particle(p, Representation(d.get_particle())); } + if (core::Provenanced::get_is_setup(d.get_particle())) { + core::Provenanced pd(d.get_particle()); + core::Provenance prov = core::create_clone(pd.get_provenance()); + core::Provenanced::setup_particle(p, prov); + } if (nd == Hierarchy()) nd = Hierarchy::setup_particle(p); using core::XYZ;