Merge branch 'main' into benchmark

.github/workflows/test_branches.yml

@@ -623,7 +623,7 @@ jobs:
         $PYTHON_EXE -m pytest -v \
             -W ignore::Warning ${{matrix.category}} \
             pyomo `pwd`/pyomo-model-libraries \
-            `pwd`/examples/pyomobook --junitxml="TEST-pyomo.xml"
+            `pwd`/examples `pwd`/doc --junitxml="TEST-pyomo.xml"
 
     - name: Run Pyomo MPI tests
       if: matrix.mpi != 0

.github/workflows/test_pr_and_main.yml

@@ -653,7 +653,7 @@ jobs:
         $PYTHON_EXE -m pytest -v \
             -W ignore::Warning ${{matrix.category}} \
             pyomo `pwd`/pyomo-model-libraries \
-            `pwd`/examples/pyomobook --junitxml="TEST-pyomo.xml"
+            `pwd`/examples `pwd`/doc --junitxml="TEST-pyomo.xml"
 
     - name: Run Pyomo MPI tests
       if: matrix.mpi != 0

.jenkins.sh

@@ -38,7 +38,7 @@ if test -z "$WORKSPACE"; then
     export WORKSPACE=`pwd`
 fi
 if test -z "$TEST_SUITES"; then
-    export TEST_SUITES="${WORKSPACE}/pyomo/pyomo ${WORKSPACE}/pyomo-model-libraries ${WORKSPACE}/pyomo/examples/pyomobook"
+    export TEST_SUITES="${WORKSPACE}/pyomo/pyomo ${WORKSPACE}/pyomo-model-libraries ${WORKSPACE}/pyomo/examples ${WORKSPACE}/pyomo/doc"
 fi
 if test -z "$SLIM"; then
     export VENV_SYSTEM_PACKAGES='--system-site-packages'

doc/OnlineDocs/Makefile

@@ -23,4 +23,4 @@ clean clean_tests:
 	@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 	@echo "Removing *.spy, *.out"
 	@find . -name \*.spy -delete
-	@find tests -name \*.out -delete
+	@find src -name \*.out -delete

doc/OnlineDocs/conf.py

@@ -26,12 +26,12 @@
 sys.path.insert(0, os.path.abspath('../..'))
 
 # -- Rebuild SPY files ----------------------------------------------------
-sys.path.insert(0, os.path.abspath('tests'))
+sys.path.insert(0, os.path.abspath('src'))
 try:
     print("Regenerating SPY files...")
     from strip_examples import generate_spy_files
 
-    generate_spy_files(os.path.abspath('tests'))
+    generate_spy_files(os.path.abspath('src'))
     generate_spy_files(
         os.path.abspath(os.path.join('library_reference', 'kernel', 'examples'))
     )

doc/OnlineDocs/developer_reference/expressions/design.rst

@@ -73,7 +73,7 @@ Expression trees can be categorized in four different ways:
 
 These three categories are illustrated with the following example:
 
-.. literalinclude:: ../../tests/expr/design_categories.spy
+.. literalinclude:: ../../src/expr/design_categories.spy
 
 The following table describes four different simple expressions
 that consist of a single model component, and it shows how they
@@ -107,7 +107,7 @@ Named expressions allow for changes to an expression after it has
 been constructed.  For example, consider the expression ``f`` defined
 with the :class:`Expression <pyomo.core.base.Expression>` component:
 
-.. literalinclude:: ../../tests/expr/design_named_expression.spy
+.. literalinclude:: ../../src/expr/design_named_expression.spy
 
 Although ``f`` is an immutable expression, whose definition is
 fixed, a sub-expressions is the named expression ``M.e``.  Named
@@ -227,7 +227,7 @@ The :data:`linear_expression <pyomo.core.expr.linear_expression>`
 object is a context manager that can be used to declare a linear sum.  For
 example, consider the following two loops:
 
-.. literalinclude:: ../../tests/expr/design_cm1.spy
+.. literalinclude:: ../../src/expr/design_cm1.spy
 
 The first apparent difference in these loops is that the value of
 ``s`` is explicitly initialized while ``e`` is initialized when the
@@ -250,7 +250,7 @@ construct different expressions with different context declarations.
 Finally, note that these context managers can be passed into the :attr:`start`
 method for the :func:`quicksum <pyomo.core.util.quicksum>` function.  For example:
 
-.. literalinclude:: ../../tests/expr/design_cm2.spy
+.. literalinclude:: ../../src/expr/design_cm2.spy
 
 This sum contains terms for ``M.x[i]`` and ``M.y[i]``.  The syntax
 in this example is not intuitive because the sum is being stored

doc/OnlineDocs/developer_reference/expressions/index.rst

@@ -21,7 +21,7 @@ nodes contain operators.  Pyomo relies on so-called magic methods
 to automate the construction of symbolic expressions.  For example,
 consider an expression ``e`` declared as follows:
 
-.. literalinclude:: ../../tests/expr/index_simple.spy
+.. literalinclude:: ../../src/expr/index_simple.spy
 
 Python determines that the magic method ``__mul__`` is called on
 the ``M.v`` object, with the argument ``2``.  This method returns

doc/OnlineDocs/developer_reference/expressions/managing.rst

@@ -23,19 +23,21 @@ mimics the Python operations used to construct an expression.  The
 :data:`verbose` flag can be set to :const:`True` to generate a
 string representation that is a nested functional form.  For example:
 
-.. literalinclude:: ../../tests/expr/managing_ex1.spy
+.. literalinclude:: ../../src/expr/managing_ex1.spy
 
 Labeler and Symbol Map
 ~~~~~~~~~~~~~~~~~~~~~~
 
-The string representation used for variables in expression can be customized to
-define different label formats.  If the :data:`labeler` option is specified, then this
-function (or class functor) is used to generate a string label used to represent the variable.  Pyomo
-defines a variety of labelers in the `pyomo.core.base.label` module.  For example, the
-:class:`NumericLabeler` defines a functor that can be used to sequentially generate
-simple labels with a prefix followed by the variable count:
+The string representation used for variables in expression can be
+customized to define different label formats.  If the :data:`labeler`
+option is specified, then this function (or class functor) is used to
+generate a string label used to represent the variable.  Pyomo defines a
+variety of labelers in the `pyomo.core.base.label` module.  For example,
+the :class:`NumericLabeler` defines a functor that can be used to
+sequentially generate simple labels with a prefix followed by the
+variable count:
 
-.. literalinclude:: ../../tests/expr/managing_ex2.spy
+.. literalinclude:: ../../src/expr/managing_ex2.spy
 
 The :data:`smap` option is used to specify a symbol map object
 (:class:`SymbolMap <pyomo.core.expr.symbol_map.SymbolMap>`), which
@@ -46,44 +48,20 @@ variables in different expressions have a consistent label in their
 associated string representations.
 
 
-Standardized String Representations
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The :data:`standardize` option can be used to re-order the string 
-representation to print polynomial terms before nonlinear terms.  By
-default, :data:`standardize` is :const:`False`, and the string
-representation reflects the order in which terms were combined to
-form the expression.  Pyomo does not guarantee that the string 
-representation exactly matches the Python expression order, since
-some simplification and re-ordering of terms is done automatically to
-improve the efficiency of expression generation.  But in most cases
-the string representation will closely correspond to the 
-Python expression order.
-
-If :data:`standardize` is :const:`True`, then the pyomo expression
-is processed to identify polynomial terms, and the string representation
-consists of the constant and linear terms followed by
-an expression that contains other nonlinear terms.  For example:
-
-.. literalinclude:: ../../tests/expr/managing_ex3.spy
-
 Other Ways to Generate String Representations
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 There are two other standard ways to generate string representations:
 
-* Call the :func:`__str__` magic method (e.g. using the Python :func:`str()` function.  This 
-  calls :func:`expression_to_string <pyomo.core.expr.expression_to_string>` with
-  the option :data:`standardize` equal to :const:`True` (see below).
+* Call the :func:`__str__` magic method (e.g. using the Python
+  :func:`str()` function.  This calls :func:`expression_to_string
+  <pyomo.core.expr.expression_to_string>`, using the default values for
+  all arguments.
 
-* Call the :func:`to_string` method on the :class:`ExpressionBase <pyomo.core.expr.ExpressionBase>` class.
-  This defaults to calling :func:`expression_to_string <pyomo.core.expr.expression_to_string>` with
-  the option :data:`standardize` equal to :const:`False` (see below).
-
-In practice, we expect at the :func:`__str__` magic method will be
-used by most users, and the standardization of the output provides
-a consistent ordering of terms that should make it easier to interpret
-expressions.
+* Call the :func:`to_string` method on the
+  :class:`ExpressionBase<pyomo.core.expr.ExpressionBase>` class.  This
+  calls :func:`expression_to_string
+  <pyomo.core.expr.expression_to_string>` and accepts the same arguments.
 
 
 Evaluating Expressions
@@ -94,21 +72,21 @@ the expression have a value.  The :func:`value <pyomo.core.expr.value>`
 function can be used to walk the expression tree and compute the
 value of an expression.  For example:
 
-.. literalinclude:: ../../tests/expr/managing_ex5.spy
+.. literalinclude:: ../../src/expr/managing_ex5.spy
 
 Additionally, expressions define the :func:`__call__` method, so the
 following is another way to compute the value of an expression:
 
-.. literalinclude:: ../../tests/expr/managing_ex6.spy
+.. literalinclude:: ../../src/expr/managing_ex6.spy
 
 If a parameter or variable is undefined, then the :func:`value
 <pyomo.core.expr.value>` function and :func:`__call__` method will
 raise an exception.  This exception can be suppressed using the
 :attr:`exception` option.  For example:
 
-.. literalinclude:: ../../tests/expr/managing_ex7.spy
+.. literalinclude:: ../../src/expr/managing_ex7.spy
 
-This option is useful in contexts where adding a try block is inconvenient 
+This option is useful in contexts where adding a try block is inconvenient
 in your modeling script.
 
 .. note::
@@ -127,10 +105,10 @@ Expression transformations sometimes need to find all nodes in an
 expression tree that are of a given type.  Pyomo contains two utility
 functions that support this functionality.  First, the
 :func:`identify_components <pyomo.core.expr.identify_components>`
-function is a generator function that walks the expression tree and yields all 
+function is a generator function that walks the expression tree and yields all
 nodes whose type is in a specified set of node types.  For example:
 
-.. literalinclude:: ../../tests/expr/managing_ex8.spy
+.. literalinclude:: ../../src/expr/managing_ex8.spy
 
 The :func:`identify_variables <pyomo.core.expr.identify_variables>`
 function is a generator function that yields all nodes that are
@@ -139,7 +117,7 @@ but this set of variable types does not need to be specified by the user.
 However, the :attr:`include_fixed` flag can be specified to omit fixed
 variables.  For example:
 
-.. literalinclude:: ../../tests/expr/managing_ex9.spy
+.. literalinclude:: ../../src/expr/managing_ex9.spy
 
 Walking an Expression Tree with a Visitor Class
 -----------------------------------------------
@@ -152,8 +130,15 @@ is computed using the values of its children.
 
 Walking an expression tree can be tricky, and the code requires intimate
 knowledge of the design of the expression system.  Pyomo includes
-several classes that define so-called visitor patterns for walking
-expression tree:
+several classes that define visitor patterns for walking expression
+tree:
+
+:class:`StreamBasedExpressionVisitor <pyomo.core.expr.StreamBasedExpressionVisitor>`
+    The most general and extensible visitor class.  This visitor
+    implements an event-based approach for walking the tree inspired by
+    the ``expat`` library for processing XML files.  The visitor has
+    seven event callbacks that users can hook into, providing very
+    fine-grained control over the expression walker.
 
 :class:`SimpleExpressionVisitor <pyomo.core.expr.SimpleExpressionVisitor>`
     A :func:`visitor` method is called for each node in the tree,
@@ -173,33 +158,39 @@ expression tree:
 
 These classes define a variety of suitable tree search methods:
 
-* :class:`SimpleExpressionVisitor <pyomo.core.expr.SimpleExpressionVisitor>`
+* :class:`StreamBasedExpressionVisitor <pyomo.core.expr.StreamBasedExpressionVisitor>`
 
-  * **xbfs**: breadth-first search where leaf nodes are immediately visited
-  * **xbfs_yield_leaves**: breadth-first search where leaf nodes are immediately visited, and the visit method yields a value
+  * ``walk_expression``: depth-first traversal of the expression tree.
 
-* :class:`ExpressionValueVisitor <pyomo.core.expr.ExpressionValueVisitor>`
+* :class:`ExpressionReplacementVisitor <pyomo.core.expr.ExpressionReplacementVisitor>`
 
-  * **dfs_postorder_stack**: postorder depth-first search using a stack
+  * ``walk_expression``: depth-first traversal of the expression tree.
 
-* :class:`ExpressionReplacementVisitor <pyomo.core.expr.ExpressionReplacementVisitor>`
+* :class:`SimpleExpressionVisitor <pyomo.core.expr.SimpleExpressionVisitor>`
 
-  * **dfs_postorder_stack**: postorder depth-first search using a stack
+  * ``xbfs``: breadth-first search where leaf nodes are immediately visited
+  * ``xbfs_yield_leaves``: breadth-first search where leaf nodes are
+    immediately visited, and the visit method yields a value
 
-.. note::
+* :class:`ExpressionValueVisitor <pyomo.core.expr.ExpressionValueVisitor>`
+
+  * ``dfs_postorder_stack``: postorder depth-first search using a
+    nonrecursive stack
 
-    The PyUtilib visitor classes define several other search methods
-    that could be used with Pyomo expressions.  But these are the 
-    only search methods currently used within Pyomo.
 
-To implement a visitor object, a user creates a subclass of one of these 
-classes.  Only one of a few methods will need to be defined to
-implement the visitor:
+To implement a visitor object, a user needs to provide specializations
+for specific events.  For legacy visitors based on the PyUtilib
+visitor pattern (e.g., :class:`SimpleExpressionVisitor` and
+:class:`ExpressionValueVisitor`), one must create a subclass of one of these
+classes and override at least one of the following:
 
 :func:`visitor`
     Defines the operation that is performed when a node is visited.  In
-    the :class:`ExpressionValueVisitor <pyomo.core.expr.ExpressionValueVisitor>` and :class:`ExpressionReplacementVisitor <pyomo.core.expr.ExpressionReplacementVisitor>` visitor classes, this 
-    method returns a value that is used by its parent node.
+    the :class:`ExpressionValueVisitor
+    <pyomo.core.expr.ExpressionValueVisitor>` and
+    :class:`ExpressionReplacementVisitor
+    <pyomo.core.expr.ExpressionReplacementVisitor>` visitor classes,
+    this method returns a value that is used by its parent node.
 
 :func:`visiting_potential_leaf`
     Checks if the search should terminate with this node.  If no,
@@ -211,9 +202,17 @@ implement the visitor:
     class.
 
 :func:`finalize`
-    This method defines the final value that is returned from the 
+    This method defines the final value that is returned from the
     visitor.  This is not normally redefined.
 
+For modern visitors based on the :class:`StreamBasedExpressionVisitor
+<pyomo.core.expr.StreamBasedExpressionVisitor>`, one can either define a
+subclass, pass the callbacks to an instance of the base class, or assign
+the callbacks as attributes on an instance of the base class.  The
+:class:`StreamBasedExpressionVisitor
+<pyomo.core.expr.StreamBasedExpressionVisitor>` provides seven
+callbacks, which are documented in the class documentation.
+
 Detailed documentation of the APIs for these methods is provided
 with the class documentation for these visitors.
 
@@ -224,14 +223,14 @@ In this example, we describe an visitor class that counts the number
 of nodes in an expression (including leaf nodes).  Consider the following
 class:
 
-.. literalinclude:: ../../tests/expr/managing_visitor1.spy
+.. literalinclude:: ../../src/expr/managing_visitor1.spy
 
-The class constructor creates a counter, and the :func:`visit` method 
+The class constructor creates a counter, and the :func:`visit` method
 increments this counter for every node that is visited.  The :func:`finalize`
 method returns the value of this counter after the tree has been walked.  The
 following function illustrates this use of this visitor class:
 
-.. literalinclude:: ../../tests/expr/managing_visitor2.spy
+.. literalinclude:: ../../src/expr/managing_visitor2.spy
 
 
 ExpressionValueVisitor Example
@@ -241,14 +240,14 @@ In this example, we describe an visitor class that clones the
 expression tree (including leaf nodes).  Consider the following
 class:
 
-.. literalinclude:: ../../tests/expr/managing_visitor3.spy
+.. literalinclude:: ../../src/expr/managing_visitor3.spy
 
 The :func:`visit` method creates a new expression node with children
 specified by :attr:`values`.  The :func:`visiting_potential_leaf`
 method performs a :func:`deepcopy` on leaf nodes, which are native
 Python types or non-expression objects.
 
-.. literalinclude:: ../../tests/expr/managing_visitor4.spy
+.. literalinclude:: ../../src/expr/managing_visitor4.spy
 
 
 ExpressionReplacementVisitor Example
@@ -259,18 +258,15 @@ variables with scaled variables, using a mutable parameter that
 can be modified later.  the following
 class:
 
-.. literalinclude:: ../../tests/expr/managing_visitor5.spy
+.. literalinclude:: ../../src/expr/managing_visitor5.spy
 
-No :func:`visit` method needs to be defined.  The
-:func:`visiting_potential_leaf` function identifies variable nodes
+No other method need to be defined.  The
+:func:`beforeChild` method identifies variable nodes
 and returns a product expression that contains a mutable parameter.
-The :class:`_LinearExpression` class has a different representation
-that embeds variables.  Hence, this class must be handled 
-in a separate condition that explicitly transforms this sub-expression.
 
-.. literalinclude:: ../../tests/expr/managing_visitor6.spy
+.. literalinclude:: ../../src/expr/managing_visitor6.spy
 
-The :func:`scale_expression` function is called with an expression and 
+The :func:`scale_expression` function is called with an expression and
 a dictionary, :attr:`scale`, that maps variable ID to model parameter.  For example:
 
-.. literalinclude:: ../../tests/expr/managing_visitor7.spy
+.. literalinclude:: ../../src/expr/managing_visitor7.spy