🔀 Merge source into doc/apply-filter-more-broadly

.circleci/config.yml

@@ -5,8 +5,8 @@ commands:
       - run:
           name: 👽️ Getting user tutorials
           command: |
-            sudo apt-get update
-            sudo apt-get -y install git pandoc
+            apt-get update
+            apt-get -y install git pandoc
             git clone --depth 1 https://github.com/FCP-INDI/C-PAC_tutorials.git docs/_sources/user/tutorials
   get-version:
     steps:
@@ -22,18 +22,21 @@ commands:
           name: ↑ Installing build dependencies
           # setuptools≤57.5.0 until traits>4.6.0 in C-PAC
           command: |
-            sudo apt-get update && sudo apt-get -y install git python3-dev graphviz graphviz-dev libgraphviz-dev pkg-config python3-sphinx
+            apt-get update && apt-get -y install git python3-dev graphviz graphviz-dev libgraphviz-dev pkg-config python3-sphinx
+            python -m venv /home/circleci/docs_env
+            source /home/circleci/docs_env/bin/activate
             pip install "setuptools<=57.5.0"
-            pip install --user -r requirements.txt
-            pip install --user -r https://raw.githubusercontent.com/${CIRCLE_PROJECT_USERNAME}/C-PAC/<< parameters.version >>/requirements.txt
-            pip install --user git+https://github.com/${CIRCLE_PROJECT_USERNAME}/C-PAC.git@<< parameters.version >>
+            pip install -r requirements.txt
+            pip install -r https://raw.githubusercontent.com/${CIRCLE_PROJECT_USERNAME}/C-PAC/<< parameters.version >>/requirements.txt
+            pip install git+https://github.com/${CIRCLE_PROJECT_USERNAME}/C-PAC.git@<< parameters.version >>
             git clone https://github.com/${CIRCLE_PROJECT_USERNAME}/C-PAC.git /home/circleci/build/C-PAC
             cd /home/circleci/build/C-PAC
             git checkout << parameters.version >>
             cd /home/circleci/build
-            python -m venv ~/simple
-            source ~/simple/bin/activate
-            pip install git+https://github.com/FCP-INDI/cpac.git@aa73ce266d3e599b6d9a1ad9721959c69dd0c4d3 semver
+            deactivate
+            python -m venv /home/circleci/simple
+            source /home/circleci/simple/bin/activate
+            pip install cpac semver
             deactivate
   
   run-cpac-commands:
@@ -42,7 +45,7 @@ commands:
       - run:
           name: 👊 Running cpac commands
           command: |
-            source ~/simple/bin/activate
+            source /home/circleci/simple/bin/activate
             cpac pull
             mkdir -p docs/_sources/user/cpac
             printf ".. code-block:: console\n\n   $ cpac --help\n\n" > docs/_sources/user/cpac/help.rst
@@ -53,6 +56,9 @@ commands:
             mkdir -p docs/_sources/user/utils
             printf "Usage: cpac utils\n\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\n.. code-block:: console\n\n   $ cpac utils --help\n\n" > docs/_sources/user/utils/help.rst
             cpac utils --help | sed -e "s/.*/   &/" >> docs/_sources/user/utils/help.rst
+            mkdir -p docs/_sources/user/group/feat/load-preset/unpaired-two
+            printf ".. code-block:: console\n\n   $ cpac group feat load-preset unpaired-two --help\n\n" > docs/_sources/user/group/feat/load-preset/unpaired-two/help.rst
+            cpac group feat load-preset unpaired-two --help | sed -e "s/.*/   &/" >> docs/_sources/user/group/feat/load-preset/unpaired-two/help.rst
             deactivate
   prep-deploy:
     steps:
@@ -64,6 +70,9 @@ commands:
           command: |
             git config --global user.email "[email protected]"
             git config --global user.name "ChildMindInstituteCNL"
+            mkdir -p /home/circleci/.ssh/
+            touch /home/circleci/.ssh/known_hosts
+            ssh-keyscan github.com >> /home/circleci/.ssh/known_hosts
       - run:
           name: 🔀 Combining new docs with existing docs
           command: |
@@ -115,7 +124,8 @@ jobs:
   build-nightly:
     working_directory: /home/circleci/build
     docker:
-      - image: cimg/python:3.10
+      - image: ghcr.io/fcp-indi/c-pac:nightly
+        user: root
     steps:
       - checkout:
           path: /home/circleci/build
@@ -125,13 +135,14 @@ jobs:
       - get-tutorials
       - run:
           name: 🏗️ Building nightly branch docs
-          command: ./bin/build nightly
+          command: source /home/circleci/docs_env/bin/activate && ./bin/build nightly
       - store-commit-message
       - persist-new-build
   build-version:
     working_directory: /home/circleci/build
     docker:
-      - image: cimg/python:3.10
+      - image: ghcr.io/fcp-indi/c-pac:nightly
+        user: root
     steps:
       - checkout:
           path: /home/circleci/build
@@ -142,7 +153,7 @@ jobs:
       - get-tutorials
       - run:
           name: 🏗️ Building version branch docs
-          command: ./bin/build ${BUILD_VERSION}
+          command: source /home/circleci/docs_env/bin/activate && ./bin/build ${BUILD_VERSION}
       - store-commit-message
       - persist-new-build
   deploy-nightly:

docs/_sources/_static/custom.css

@@ -18,3 +18,9 @@
   width: 100%;
   height: auto;
 }
+
+/* empty content highlight divs ―
+   see 'hideEmptyHighlights.js' */
+div[class*=" highlight"].hidden, div[class^=highlight].hidden {
+	display: none;
+}

docs/_sources/_static/hideEmptyHighlights.js

@@ -0,0 +1,9 @@
+/* Hide highlight containers only empty spans */
+document.addEventListener("DOMContentLoaded", function() {
+  const highlightDivs = document.querySelectorAll('div[class^="highlight"], div[class*=" highlight"]');
+  highlightDivs.forEach(function(highlightDiv) {
+    if (highlightDiv.textContent.trim() === '') {
+      highlightDiv.classList.add('hidden');
+    }
+  });
+});

docs/_sources/conf.py

@@ -459,6 +459,7 @@ def _unireplace(release_note, unireplace):
     'custom.css',
 ]
 html_js_files = [
+    'hideEmptyHighlights.js',
     ('versionList.js', {'defer': 'defer'})]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,

docs/_sources/developer/nodes.rst

@@ -7,9 +7,9 @@ Nodes
 
 .. _n_cpus:
 
-A Nipype :py:class:`~nipype.pipeline.engine.nodes.Node` has an initialization parameter ``mem_gb`` that differs from the :doc:`commandline option </user/run/help>` ``--mem_gb``. While the commandline option is **a limit**, the Node initialization parameter is **an estimate** of the most memory that Node will consume when run. The Node parameter is not a limit; rather, this value is used to allocate system resources at runtime.
+A Nipype :py:class:`~nipype.pipeline.engine.nodes.Node` has an initialization parameter ``mem_gb`` that differs from the :doc:`commandline option </user/run/help>` ``--mem-gb``. While the commandline option is **a limit**, the Node initialization parameter is **an estimate** of the most memory that Node will consume when run. The Node parameter is not a limit; rather, this value is used to allocate system resources at runtime.
 
-Conversely, the commandline option ``--n_cpus`` is **a limit** and the Node initialization parameter ``n_procs`` is **also a limit** of the maximum number of threads a Node will be permmitted to consume.
+Conversely, the commandline option ``--n-cpus`` is **a limit** and the Node initialization parameter ``n_procs`` is **also a limit** of the maximum number of threads a Node will be permmitted to consume.
 
 C-PAC automatically creates a JSON-like file called ``callback.log`` (via the function :py:func:`~CPAC.utils.monitoring.log_nodes_cb`) when running. This file includes for each Node:
 
@@ -18,7 +18,7 @@ C-PAC automatically creates a JSON-like file called ``callback.log`` (via the fu
    * specified maximum number of threads per Node, and
    * threads used at runtime.
 
-A ``callback.log`` can be provided to the pipeline configuration file (see :doc:`/user/compute_config`) or with the commandline flag ``--runtime_usage``. If a callback log is provided in the pipeline configuration, nodes with names that match nodes recorded in that pipeline log will have their memory estimates overridden by the values in the callback log plus a buffer percent (provided with the ``--runtime_buffer`` flag or in the pipeline configuration file).
+A ``callback.log`` can be provided to the pipeline configuration file (see :doc:`/user/compute_config`) or with the commandline flag ``--runtime-usage``. If a callback log is provided in the pipeline configuration, nodes with names that match nodes recorded in that pipeline log will have their memory estimates overridden by the values in the callback log plus a buffer percent (provided with the ``--runtime-buffer`` flag or in the pipeline configuration file).
 
 When a developer creates or modifies a Node in C-PAC, a ``mem_gb`` and ``n_procs`` argument should be provided unless the respective defaults of 0.2 and None (number of available system cores) are expected to be sufficient. When testing, the ``mem_gb`` and ``n_procs`` arguments should be adjusted if the observed memory and/or thread usage of a Node exceeds the estimate.
 

docs/_sources/developer/workflows/network_centrality.rst

@@ -1,5 +1,20 @@
 Network Centrality
 ==================
 
-.. automodule:: CPAC.network_centrality
-    :members:
+.. automodule:: CPAC.network_centrality.network_centrality
+   :special-members: __init__
+   :members:
+   :private-members:
+   :undoc-members:
+
+.. automodule:: CPAC.network_centrality.pipeline
+   :special-members: __init__
+   :members:
+   :private-members:
+   :undoc-members:
+
+.. automodule:: CPAC.network_centrality.utils
+   :special-members: __init__
+   :members:
+   :private-members:
+   :undoc-members:

docs/_sources/user/compute_config.rst

@@ -20,8 +20,8 @@ Computer Settings
 
         `Open image <../_static/flowcharts/observed-usage.svg>`_
 
-    #. **Callback log - [text]:** The path to a callback log file from a previous run, including any resource-management parameters that will be applied in this run, like ``n_cpus`` and ``num_ants_threads``. This file is used override memory estimates with previously observed memory usage. Can be overridden with the commandline flag ``--runtime_usage``.
-    #. **Buffer - [percent]:** A percent of the previously observed memory usage that is to be added to the memory estimate. Default: 10. Can be overridden with the commandline flag ``--runtime_buffer``.
+    #. **Callback log - [text]:** The path to a callback log file from a previous run, including any resource-management parameters that will be applied in this run, like ``n_cpus`` and ``num_ants_threads``. This file is used override memory estimates with previously observed memory usage. Can be overridden with the commandline flag ``--runtime-usage``.
+    #. **Buffer - [percent]:** A percent of the previously observed memory usage that is to be added to the memory estimate. Default: 10. Can be overridden with the commandline flag ``--runtime-buffer``.
 
 #. **Number of Participants to Run Simultaneously - [integer]:** This number depends on computing resources.
 

docs/_sources/user/cpac.rst

@@ -3,12 +3,26 @@ cpac (Python package)
 
 `cpac <https://pypi.org/project/cpac/>`_ is available so that you can easily run analyses without needing interact with the container platform that allows you to run C-PAC without installing all of the underlying software.
 
-cpac requires Python 3.6 or greater. To get cpac, simply
+``cpac`` requires Python 3.8 or greater. To get cpac, simply
 
 .. code-block:: console
 
     pip install cpac
 
+Run extra packages with cpac
+````````````````````````````
+Beginning with v1.8.5, ``cpac`` can run |ba_timeseries_gradients|_ and/or |tsconcat|_ in addition to C-PAC. If either package is already installed, ``cpac`` will use the installed version. You can install them with the relevant extras, like one of the following lines:
+
+.. code-block:: console
+
+    pip install cpac[ba_timeseries_gradients]
+    pip install cpac[tsconcat]
+    pip install cpac[ba_timeseries_gradients,tsconcat]
+
+.. note::
+
+    ``ba_timeseries_gradients`` and ``tsconcat`` may have more specific dependency requirements than C-PAC. If you run into unclear installation issues when installing these extras, please see the documentation for those packages.
+
 Download / Upgrade C-PAC with cpac
 ``````````````````````````````````
 
@@ -88,32 +102,42 @@ To run C-PAC with a pipeline configuration file other than one of the pre-config
 
 .. code-block:: console
 
-    cpac run /Users/You/local_bids_data /Users/You/some_folder_for_outputs participant --pipeline_file /Users/You/Documents/pipeline_config.yml
+    cpac run /Users/You/local_bids_data /Users/You/some_folder_for_outputs participant --pipeline-file /Users/You/Documents/pipeline_config.yml
 
 Finally, to run C-PAC with a specific data configuration file (instead of providing a BIDS data directory):
 
 .. code-block:: console
 
-    cpac run /Users/You/any_directory /Users/You/some_folder_for_outputs participant --data_config_file /Users/You/Documents/data_config.yml
+    cpac run /Users/You/any_directory /Users/You/some_folder_for_outputs participant --data-config-file /Users/You/Documents/data_config.yml
 
-Note: we are still providing the postionally-required ``bids_dir`` input parameter. However C-PAC will not look for data in this directory when you provide a data configuration YAML with the ``--data_config_file`` flag. Providing ``.`` or ``$PWD`` will simply pass the present working directory. In addition, if the dataset in your data configuration file is not in BIDS format, just make sure to add the ``--skip_bids_validator`` flag at the end of your command to bypass the BIDS validation process.
+Note: we are still providing the postionally-required ``bids_dir`` input parameter. However C-PAC will not look for data in this directory when you provide a data configuration YAML with the ``--data-config-file`` flag. Providing ``.`` or ``$PWD`` will simply pass the present working directory. In addition, if the dataset in your data configuration file is not in BIDS format, just make sure to add the ``--skip-bids-validator`` flag at the end of your command to bypass the BIDS validation process.
 
 The full list of parameters and options that can be passed to C-PAC are shown below:
 
 .. include:: /user/run/help.rst
 
 .. include:: /user/utils/help.rst
 
-Note that any of the optional arguments above will over-ride any pipeline settings in the default pipeline or in the pipeline configuration file you provide via the ``--pipeline_file`` parameter.
+Note that any of the optional arguments above will over-ride any pipeline settings in the default pipeline or in the pipeline configuration file you provide via the ``--pipeline-file`` parameter.
 
 **Further usage notes:**
 
-* You can run only anatomical preprocessing easily, without modifying your data or pipeline configuration files, by providing the ``--anat_only`` flag.
+* You can run only anatomical preprocessing easily, without modifying your data or pipeline configuration files, by providing the ``--anat-only`` flag.
 
-* As stated, the default behavior is to read data that is organized in the BIDS format. This includes data that is in Amazon AWS S3 by using the format ``s3://<bucket_name>/<bids_dir>`` for the ``bids_dir`` command line argument. Outputs can be written to S3 using the same format for the ``output_dir``. Credentials for accessing these buckets can be specified on the command line (using ``--aws_input_creds`` or ``--aws_output_creds``).
+* As stated, the default behavior is to read data that is organized in the BIDS format. This includes data that is in Amazon AWS S3 by using the format ``s3://<bucket_name>/<bids_dir>`` for the ``bids_dir`` command line argument. Outputs can be written to S3 using the same format for the ``output_dir``. Credentials for accessing these buckets can be specified on the command line (using ``--aws-input-creds`` or ``--aws-output-creds``).
 
-* When the app is run, a data configuration file is written to the working directory. This directory can be specified with ``--working_dir`` or the directory from which you run ``cpac`` will be used. This file can be passed into subsequent runs, which avoids the overhead of re-parsing the BIDS input directory on each run (i.e. for cluster or cloud runs). These files can be generated without executing the C-PAC pipeline using the ``test_run`` command line argument.
+* When the app is run, a data configuration file is written to the working directory. This directory can be specified with ``--working-dir`` or the directory from which you run ``cpac`` will be used. This file can be passed into subsequent runs, which avoids the overhead of re-parsing the BIDS input directory on each run (i.e. for cluster or cloud runs). These files can be generated without executing the C-PAC pipeline using ``test_config`` as the analysis level.
 
 * The ``participant_label`` and ``participant_ndx`` arguments allow the user to specify which of the many datasets should be processed, which is useful when parallelizing the run of multiple participants.
 
 * If you want to pass runtime options to your container plaform (Docker or Singularity), you can pass them with ``-o`` or ``--container_options``.
+
+.. TODO: Update cpac to handle `-`s and `_`s like C-PAC
+
+.. |ba_timeseries_gradients| replace:: ``ba_timeseries_gradients``
+
+.. _ba_timeseries_gradients: https://cmi-dair.github.io/ba-timeseries-gradients/ba_timeseries_gradients.html
+
+.. |tsconcat| replace:: ``tsconcat``
+
+.. _tsconcat: https://cmi-dair.github.io/tsconcat/tsconcat.html