From a19df5badcc0887e95ec02f4ebe4671c6d12ba5b Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 30 Aug 2024 14:30:11 -0700 Subject: [PATCH 01/70] switching this lesson to the new Carpentries Workbench Template --- .github/workflows/README.md | 198 ++++++ .github/workflows/pr-close-signal.yaml | 23 + .github/workflows/pr-comment.yaml | 185 ++++++ .github/workflows/pr-post-remove-branch.yaml | 32 + .github/workflows/pr-preflight.yaml | 39 ++ .github/workflows/pr-receive.yaml | 131 ++++ .github/workflows/sandpaper-main.yaml | 61 ++ .github/workflows/sandpaper-version.txt | 1 + .github/workflows/update-cache.yaml | 125 ++++ .github/workflows/update-workflows.yaml | 66 ++ .gitignore | 66 +- CITATION.cff | 22 + CODE_OF_CONDUCT.md | 8 +- CONTRIBUTING.md | 152 +++-- FIXME.Rproj | 19 + LICENSE.md | 104 ++-- _config.yml | 104 ---- _episodes/01-intro.md | 164 ----- _episodes/05-loops.md | 335 ----------- _episodes/07-commandargs.md | 105 ---- _episodes/11-parallel-intro.md | 107 ---- _episodes/12-fire-forget-tasks.md | 401 ------------- _extras/.gitkeep | 0 _extras/about.md | 6 - _extras/discuss.md | 5 - _extras/figures.md | 6 - _extras/guide.md | 13 - _includes/all_figures.html | 1 - _includes/links.md | 46 -- aio.md | 38 -- bin/chunk-options.R | 70 --- bin/extract_figures.py | 98 --- bin/generate_md_episodes.R | 71 --- bin/knit_lessons.sh | 8 - bin/lesson_check.py | 564 ------------------ bin/lesson_initialize.py | 52 -- bin/markdown_ast.rb | 13 - bin/repo_check.py | 180 ------ bin/run-make-docker-serve.sh | 10 - bin/test_lesson_check.py | 23 - bin/util.py | 188 ------ bin/workshop_check.py | 418 ------------- code/.gitkeep | 0 config.yaml | 97 +++ data/.gitkeep | 0 {_episodes => episodes}/.gitkeep | 0 episodes/01-intro.md | 143 +++++ {_episodes => episodes}/02-variables.md | 181 +++--- {_episodes => episodes}/03-ranges-arrays.md | 151 +++-- {_episodes => episodes}/04-conditionals.md | 135 ++--- episodes/05-loops.md | 331 ++++++++++ {_episodes => episodes}/06-procedures.md | 53 +- episodes/07-commandargs.md | 101 ++++ {_episodes => episodes}/08-timing.md | 74 ++- episodes/11-parallel-intro.md | 101 ++++ episodes/12-fire-forget-tasks.md | 392 ++++++++++++ {_episodes => episodes}/13-synchronization.md | 235 ++++---- .../14-parallel-case-study.md | 244 ++++---- {_episodes => episodes}/21-locales.md | 196 +++--- {_episodes => episodes}/22-domains.md | 463 +++++++------- fig/.gitkeep | 0 files/.gitkeep | 0 files/linux-cloud.jpg | Bin 24843 -> 0 bytes index.md | 30 +- instructors/instructor-notes.md | 5 + learners/reference.md | 8 + learners/setup.md | 55 ++ lesson-outline.md | 86 --- links.md | 10 + profiles/learner-profiles.md | 5 + reference.md | 7 - setup.md | 134 ----- site/README.md | 2 + 73 files changed, 3136 insertions(+), 4361 deletions(-) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/pr-close-signal.yaml create mode 100644 .github/workflows/pr-comment.yaml create mode 100644 .github/workflows/pr-post-remove-branch.yaml create mode 100644 .github/workflows/pr-preflight.yaml create mode 100644 .github/workflows/pr-receive.yaml create mode 100644 .github/workflows/sandpaper-main.yaml create mode 100644 .github/workflows/sandpaper-version.txt create mode 100644 .github/workflows/update-cache.yaml create mode 100644 .github/workflows/update-workflows.yaml create mode 100644 CITATION.cff create mode 100644 FIXME.Rproj delete mode 100644 _config.yml delete mode 100644 _episodes/01-intro.md delete mode 100644 _episodes/05-loops.md delete mode 100644 _episodes/07-commandargs.md delete mode 100644 _episodes/11-parallel-intro.md delete mode 100644 _episodes/12-fire-forget-tasks.md delete mode 100644 _extras/.gitkeep delete mode 100644 _extras/about.md delete mode 100644 _extras/discuss.md delete mode 100644 _extras/figures.md delete mode 100644 _extras/guide.md delete mode 100644 _includes/all_figures.html delete mode 100644 _includes/links.md delete mode 100644 aio.md delete mode 100644 bin/chunk-options.R delete mode 100755 bin/extract_figures.py delete mode 100644 bin/generate_md_episodes.R delete mode 100755 bin/knit_lessons.sh delete mode 100755 bin/lesson_check.py delete mode 100755 bin/lesson_initialize.py delete mode 100755 bin/markdown_ast.rb delete mode 100755 bin/repo_check.py delete mode 100644 bin/run-make-docker-serve.sh delete mode 100755 bin/test_lesson_check.py delete mode 100644 bin/util.py delete mode 100755 bin/workshop_check.py delete mode 100644 code/.gitkeep create mode 100644 config.yaml delete mode 100644 data/.gitkeep rename {_episodes => episodes}/.gitkeep (100%) create mode 100644 episodes/01-intro.md rename {_episodes => episodes}/02-variables.md (58%) rename {_episodes => episodes}/03-ranges-arrays.md (66%) rename {_episodes => episodes}/04-conditionals.md (62%) create mode 100644 episodes/05-loops.md rename {_episodes => episodes}/06-procedures.md (65%) create mode 100644 episodes/07-commandargs.md rename {_episodes => episodes}/08-timing.md (68%) create mode 100644 episodes/11-parallel-intro.md create mode 100644 episodes/12-fire-forget-tasks.md rename {_episodes => episodes}/13-synchronization.md (61%) rename {_episodes => episodes}/14-parallel-case-study.md (62%) rename {_episodes => episodes}/21-locales.md (51%) rename {_episodes => episodes}/22-domains.md (68%) delete mode 100644 fig/.gitkeep delete mode 100644 files/.gitkeep delete mode 100644 files/linux-cloud.jpg create mode 100644 instructors/instructor-notes.md create mode 100644 learners/reference.md create mode 100644 learners/setup.md delete mode 100644 lesson-outline.md create mode 100644 links.md create mode 100644 profiles/learner-profiles.md delete mode 100644 reference.md delete mode 100644 setup.md create mode 100644 site/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..7076ddd --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,198 @@ +# Carpentries Workflows + +This directory contains workflows to be used for Lessons using the {sandpaper} +lesson infrastructure. Two of these workflows require R (`sandpaper-main.yaml` +and `pr-receive.yaml`) and the rest are bots to handle pull request management. + +These workflows will likely change as {sandpaper} evolves, so it is important to +keep them up-to-date. To do this in your lesson you can do the following in your +R console: + +```r +# Install/Update sandpaper +options(repos = c(carpentries = "https://carpentries.r-universe.dev/", + CRAN = "https://cloud.r-project.org")) +install.packages("sandpaper") + +# update the workflows in your lesson +library("sandpaper") +update_github_workflows() +``` + +Inside this folder, you will find a file called `sandpaper-version.txt`, which +will contain a version number for sandpaper. This will be used in the future to +alert you if a workflow update is needed. + +What follows are the descriptions of the workflow files: + +## Deployment + +### 01 Build and Deploy (sandpaper-main.yaml) + +This is the main driver that will only act on the main branch of the repository. +This workflow does the following: + + 1. checks out the lesson + 2. provisions the following resources + - R + - pandoc + - lesson infrastructure (stored in a cache) + - lesson dependencies if needed (stored in a cache) + 3. builds the lesson via `sandpaper:::ci_deploy()` + +#### Caching + +This workflow has two caches; one cache is for the lesson infrastructure and +the other is for the the lesson dependencies if the lesson contains rendered +content. These caches are invalidated by new versions of the infrastructure and +the `renv.lock` file, respectively. If there is a problem with the cache, +manual invaliation is necessary. You will need maintain access to the repository +and you can either go to the actions tab and [click on the caches button to find +and invalidate the failing cache](https://github.blog/changelog/2022-10-20-manage-caches-in-your-actions-workflows-from-web-interface/) +or by setting the `CACHE_VERSION` secret to the current date (which will +invalidate all of the caches). + +## Updates + +### Setup Information + +These workflows run on a schedule and at the maintainer's request. Because they +create pull requests that update workflows/require the downstream actions to run, +they need a special repository/organization secret token called +`SANDPAPER_WORKFLOW` and it must have the `public_repo` and `workflow` scope. + +This can be an individual user token, OR it can be a trusted bot account. If you +have a repository in one of the official Carpentries accounts, then you do not +need to worry about this token being present because the Carpentries Core Team +will take care of supplying this token. + +If you want to use your personal account: you can go to + +to create a token. Once you have created your token, you should copy it to your +clipboard and then go to your repository's settings > secrets > actions and +create or edit the `SANDPAPER_WORKFLOW` secret, pasting in the generated token. + +If you do not specify your token correctly, the runs will not fail and they will +give you instructions to provide the token for your repository. + +### 02 Maintain: Update Workflow Files (update-workflow.yaml) + +The {sandpaper} repository was designed to do as much as possible to separate +the tools from the content. For local builds, this is absolutely true, but +there is a minor issue when it comes to workflow files: they must live inside +the repository. + +This workflow ensures that the workflow files are up-to-date. The way it work is +to download the update-workflows.sh script from GitHub and run it. The script +will do the following: + +1. check the recorded version of sandpaper against the current version on github +2. update the files if there is a difference in versions + +After the files are updated, if there are any changes, they are pushed to a +branch called `update/workflows` and a pull request is created. Maintainers are +encouraged to review the changes and accept the pull request if the outputs +are okay. + +This update is run weekly or on demand. + +### 03 Maintain: Update Package Cache (update-cache.yaml) + +For lessons that have generated content, we use {renv} to ensure that the output +is stable. This is controlled by a single lockfile which documents the packages +needed for the lesson and the version numbers. This workflow is skipped in +lessons that do not have generated content. + +Because the lessons need to remain current with the package ecosystem, it's a +good idea to make sure these packages can be updated periodically. The +update cache workflow will do this by checking for updates, applying them in a +branch called `updates/packages` and creating a pull request with _only the +lockfile changed_. + +From here, the markdown documents will be rebuilt and you can inspect what has +changed based on how the packages have updated. + +## Pull Request and Review Management + +Because our lessons execute code, pull requests are a secruity risk for any +lesson and thus have security measures associted with them. **Do not merge any +pull requests that do not pass checks and do not have bots commented on them.** + +This series of workflows all go together and are described in the following +diagram and the below sections: + +![Graph representation of a pull request](https://carpentries.github.io/sandpaper/articles/img/pr-flow.dot.svg) + +### Pre Flight Pull Request Validation (pr-preflight.yaml) + +This workflow runs every time a pull request is created and its purpose is to +validate that the pull request is okay to run. This means the following things: + +1. The pull request does not contain modified workflow files +2. If the pull request contains modified workflow files, it does not contain + modified content files (such as a situation where @carpentries-bot will + make an automated pull request) +3. The pull request does not contain an invalid commit hash (e.g. from a fork + that was made before a lesson was transitioned from styles to use the + workbench). + +Once the checks are finished, a comment is issued to the pull request, which +will allow maintainers to determine if it is safe to run the +"Receive Pull Request" workflow from new contributors. + +### Receive Pull Request (pr-receive.yaml) + +**Note of caution:** This workflow runs arbitrary code by anyone who creates a +pull request. GitHub has safeguarded the token used in this workflow to have no +priviledges in the repository, but we have taken precautions to protect against +spoofing. + +This workflow is triggered with every push to a pull request. If this workflow +is already running and a new push is sent to the pull request, the workflow +running from the previous push will be cancelled and a new workflow run will be +started. + +The first step of this workflow is to check if it is valid (e.g. that no +workflow files have been modified). If there are workflow files that have been +modified, a comment is made that indicates that the workflow is not run. If +both a workflow file and lesson content is modified, an error will occurr. + +The second step (if valid) is to build the generated content from the pull +request. This builds the content and uploads three artifacts: + +1. The pull request number (pr) +2. A summary of changes after the rendering process (diff) +3. The rendered files (build) + +Because this workflow builds generated content, it follows the same general +process as the `sandpaper-main` workflow with the same caching mechanisms. + +The artifacts produced are used by the next workflow. + +### Comment on Pull Request (pr-comment.yaml) + +This workflow is triggered if the `pr-receive.yaml` workflow is successful. +The steps in this workflow are: + +1. Test if the workflow is valid and comment the validity of the workflow to the + pull request. +2. If it is valid: create an orphan branch with two commits: the current state + of the repository and the proposed changes. +3. If it is valid: update the pull request comment with the summary of changes + +Importantly: if the pull request is invalid, the branch is not created so any +malicious code is not published. + +From here, the maintainer can request changes from the author and eventually +either merge or reject the PR. When this happens, if the PR was valid, the +preview branch needs to be deleted. + +### Send Close PR Signal (pr-close-signal.yaml) + +Triggered any time a pull request is closed. This emits an artifact that is the +pull request number for the next action + +### Remove Pull Request Branch (pr-post-remove-branch.yaml) + +Tiggered by `pr-close-signal.yaml`. This removes the temporary branch associated with +the pull request (if it was created). diff --git a/.github/workflows/pr-close-signal.yaml b/.github/workflows/pr-close-signal.yaml new file mode 100644 index 0000000..d20a299 --- /dev/null +++ b/.github/workflows/pr-close-signal.yaml @@ -0,0 +1,23 @@ +name: "Bot: Send Close Pull Request Signal" + +on: + pull_request: + types: + [closed] + +jobs: + send-close-signal: + name: "Send closing signal" + runs-on: ubuntu-latest + if: ${{ github.event.action == 'closed' }} + steps: + - name: "Create PRtifact" + run: | + mkdir -p ./pr + printf ${{ github.event.number }} > ./pr/NUM + - name: Upload Diff + uses: actions/upload-artifact@v4 + with: + name: pr + path: ./pr + diff --git a/.github/workflows/pr-comment.yaml b/.github/workflows/pr-comment.yaml new file mode 100644 index 0000000..8a2bd3c --- /dev/null +++ b/.github/workflows/pr-comment.yaml @@ -0,0 +1,185 @@ +name: "Bot: Comment on the Pull Request" + +# read-write repo token +# access to secrets +on: + workflow_run: + workflows: ["Receive Pull Request"] + types: + - completed + +concurrency: + group: pr-${{ github.event.workflow_run.pull_requests[0].number }} + cancel-in-progress: true + + +jobs: + # Pull requests are valid if: + # - they match the sha of the workflow run head commit + # - they are open + # - no .github files were committed + test-pr: + name: "Test if pull request is valid" + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + payload: ${{ steps.check-pr.outputs.payload }} + number: ${{ steps.get-pr.outputs.NUM }} + msg: ${{ steps.check-pr.outputs.MSG }} + steps: + - name: 'Download PR artifact' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'pr' + + - name: "Get PR Number" + if: ${{ steps.dl.outputs.success == 'true' }} + id: get-pr + run: | + unzip pr.zip + echo "NUM=$(<./NR)" >> $GITHUB_OUTPUT + + - name: "Fail if PR number was not present" + id: bad-pr + if: ${{ steps.dl.outputs.success != 'true' }} + run: | + echo '::error::A pull request number was not recorded. The pull request that triggered this workflow is likely malicious.' + exit 1 + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "Check PR" + id: check-pr + if: ${{ steps.dl.outputs.success == 'true' }} + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ steps.get-pr.outputs.NUM }} + sha: ${{ github.event.workflow_run.head_sha }} + headroom: 3 # if it's within the last three commits, we can keep going, because it's likely rapid-fire + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + fail_on_error: true + + # Create an orphan branch on this repository with two commits + # - the current HEAD of the md-outputs branch + # - the output from running the current HEAD of the pull request through + # the md generator + create-branch: + name: "Create Git Branch" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + NR: ${{ needs.test-pr.outputs.number }} + permissions: + contents: write + steps: + - name: 'Checkout md outputs' + uses: actions/checkout@v4 + with: + ref: md-outputs + path: built + fetch-depth: 1 + + - name: 'Download built markdown' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'built' + + - if: ${{ steps.dl.outputs.success == 'true' }} + run: unzip built.zip + + - name: "Create orphan and push" + if: ${{ steps.dl.outputs.success == 'true' }} + run: | + cd built/ + git config --local user.email "actions@github.com" + git config --local user.name "GitHub Actions" + CURR_HEAD=$(git rev-parse HEAD) + git checkout --orphan md-outputs-PR-${NR} + git add -A + git commit -m "source commit: ${CURR_HEAD}" + ls -A | grep -v '^.git$' | xargs -I _ rm -r '_' + cd .. + unzip -o -d built built.zip + cd built + git add -A + git commit --allow-empty -m "differences for PR #${NR}" + git push -u --force --set-upstream origin md-outputs-PR-${NR} + + # Comment on the Pull Request with a link to the branch and the diff + comment-pr: + name: "Comment on Pull Request" + needs: [test-pr, create-branch] + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + NR: ${{ needs.test-pr.outputs.number }} + permissions: + pull-requests: write + steps: + - name: 'Download comment artifact' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'diff' + + - if: ${{ steps.dl.outputs.success == 'true' }} + run: unzip ${{ github.workspace }}/diff.zip + + - name: "Comment on PR" + id: comment-diff + if: ${{ steps.dl.outputs.success == 'true' }} + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ env.NR }} + path: ${{ github.workspace }}/diff.md + + # Comment if the PR is open and matches the SHA, but the workflow files have + # changed + comment-changed-workflow: + name: "Comment if workflow files have changed" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ always() && needs.test-pr.outputs.is_valid == 'false' }} + env: + NR: ${{ github.event.workflow_run.pull_requests[0].number }} + body: ${{ needs.test-pr.outputs.msg }} + permissions: + pull-requests: write + steps: + - name: 'Check for spoofing' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'built' + + - name: 'Alert if spoofed' + id: spoof + if: ${{ steps.dl.outputs.success == 'true' }} + run: | + echo 'body<> $GITHUB_ENV + echo '' >> $GITHUB_ENV + echo '## :x: DANGER :x:' >> $GITHUB_ENV + echo 'This pull request has modified workflows that created output. Close this now.' >> $GITHUB_ENV + echo '' >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + + - name: "Comment on PR" + id: comment-diff + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ env.NR }} + body: ${{ env.body }} + diff --git a/.github/workflows/pr-post-remove-branch.yaml b/.github/workflows/pr-post-remove-branch.yaml new file mode 100644 index 0000000..62c2e98 --- /dev/null +++ b/.github/workflows/pr-post-remove-branch.yaml @@ -0,0 +1,32 @@ +name: "Bot: Remove Temporary PR Branch" + +on: + workflow_run: + workflows: ["Bot: Send Close Pull Request Signal"] + types: + - completed + +jobs: + delete: + name: "Delete branch from Pull Request" + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + permissions: + contents: write + steps: + - name: 'Download artifact' + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: pr + - name: "Get PR Number" + id: get-pr + run: | + unzip pr.zip + echo "NUM=$(<./NUM)" >> $GITHUB_OUTPUT + - name: 'Remove branch' + uses: carpentries/actions/remove-branch@main + with: + pr: ${{ steps.get-pr.outputs.NUM }} diff --git a/.github/workflows/pr-preflight.yaml b/.github/workflows/pr-preflight.yaml new file mode 100644 index 0000000..d0d7420 --- /dev/null +++ b/.github/workflows/pr-preflight.yaml @@ -0,0 +1,39 @@ +name: "Pull Request Preflight Check" + +on: + pull_request_target: + branches: + ["main"] + types: + ["opened", "synchronize", "reopened"] + +jobs: + test-pr: + name: "Test if pull request is valid" + if: ${{ github.event.action != 'closed' }} + runs-on: ubuntu-latest + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + permissions: + pull-requests: write + steps: + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "Check PR" + id: check-pr + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ github.event.number }} + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + fail_on_error: true + - name: "Comment result of validation" + id: comment-diff + if: ${{ always() }} + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ github.event.number }} + body: ${{ steps.check-pr.outputs.MSG }} diff --git a/.github/workflows/pr-receive.yaml b/.github/workflows/pr-receive.yaml new file mode 100644 index 0000000..2d7d5db --- /dev/null +++ b/.github/workflows/pr-receive.yaml @@ -0,0 +1,131 @@ +name: "Receive Pull Request" + +on: + pull_request: + types: + [opened, synchronize, reopened] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + test-pr: + name: "Record PR number" + if: ${{ github.event.action != 'closed' }} + runs-on: ubuntu-latest + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + steps: + - name: "Record PR number" + id: record + if: ${{ always() }} + run: | + echo ${{ github.event.number }} > ${{ github.workspace }}/NR # 2022-03-02: artifact name fixed to be NR + - name: "Upload PR number" + id: upload + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: pr + path: ${{ github.workspace }}/NR + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "echo output" + run: | + echo "${{ steps.hash.outputs.json }}" + - name: "Check PR" + id: check-pr + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ github.event.number }} + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + + build-md-source: + name: "Build markdown source files if valid" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + CHIVE: ${{ github.workspace }}/site/chive + PR: ${{ github.workspace }}/site/pr + MD: ${{ github.workspace }}/site/built + steps: + - name: "Check Out Main Branch" + uses: actions/checkout@v4 + + - name: "Check Out Staging Branch" + uses: actions/checkout@v4 + with: + ref: md-outputs + path: ${{ env.MD }} + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Set up Pandoc" + uses: r-lib/actions/setup-pandoc@v2 + + - name: "Setup Lesson Engine" + uses: carpentries/actions/setup-sandpaper@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Setup Package Cache" + uses: carpentries/actions/setup-lesson-deps@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Validate and Build Markdown" + id: build-site + run: | + sandpaper::package_cache_trigger(TRUE) + sandpaper::validate_lesson(path = '${{ github.workspace }}') + sandpaper:::build_markdown(path = '${{ github.workspace }}', quiet = FALSE) + shell: Rscript {0} + + - name: "Generate Artifacts" + id: generate-artifacts + run: | + sandpaper:::ci_bundle_pr_artifacts( + repo = '${{ github.repository }}', + pr_number = '${{ github.event.number }}', + path_md = '${{ env.MD }}', + path_pr = '${{ env.PR }}', + path_archive = '${{ env.CHIVE }}', + branch = 'md-outputs' + ) + shell: Rscript {0} + + - name: "Upload PR" + uses: actions/upload-artifact@v4 + with: + name: pr + path: ${{ env.PR }} + + - name: "Upload Diff" + uses: actions/upload-artifact@v4 + with: + name: diff + path: ${{ env.CHIVE }} + retention-days: 1 + + - name: "Upload Build" + uses: actions/upload-artifact@v4 + with: + name: built + path: ${{ env.MD }} + retention-days: 1 + + - name: "Teardown" + run: sandpaper::reset_site() + shell: Rscript {0} diff --git a/.github/workflows/sandpaper-main.yaml b/.github/workflows/sandpaper-main.yaml new file mode 100644 index 0000000..a4f8dc4 --- /dev/null +++ b/.github/workflows/sandpaper-main.yaml @@ -0,0 +1,61 @@ +name: "01 Build and Deploy Site" + +on: + push: + branches: + - main + - master + schedule: + - cron: '0 0 * * 2' + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build?' + required: true + default: 'Maintainer (via GitHub)' + reset: + description: 'Reset cached markdown files' + required: false + default: false + type: boolean +jobs: + full-build: + name: "Build Full Site" + runs-on: ubuntu-latest + permissions: + checks: write + contents: write + pages: write + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + steps: + + - name: "Checkout Lesson" + uses: actions/checkout@v4 + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Set up Pandoc" + uses: r-lib/actions/setup-pandoc@v2 + + - name: "Setup Lesson Engine" + uses: carpentries/actions/setup-sandpaper@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Setup Package Cache" + uses: carpentries/actions/setup-lesson-deps@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Deploy Site" + run: | + reset <- "${{ github.event.inputs.reset }}" == "true" + sandpaper::package_cache_trigger(TRUE) + sandpaper:::ci_deploy(reset = reset) + shell: Rscript {0} diff --git a/.github/workflows/sandpaper-version.txt b/.github/workflows/sandpaper-version.txt new file mode 100644 index 0000000..c3f6580 --- /dev/null +++ b/.github/workflows/sandpaper-version.txt @@ -0,0 +1 @@ +0.16.6 diff --git a/.github/workflows/update-cache.yaml b/.github/workflows/update-cache.yaml new file mode 100644 index 0000000..08ea9c9 --- /dev/null +++ b/.github/workflows/update-cache.yaml @@ -0,0 +1,125 @@ +name: "03 Maintain: Update Package Cache" + +on: + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build (enter github username to tag yourself)?' + required: true + default: 'monthly run' + schedule: + # Run every tuesday + - cron: '0 0 * * 2' + +jobs: + preflight: + name: "Preflight Check" + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.check.outputs.ok }} + steps: + - id: check + run: | + if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then + echo "ok=true" >> $GITHUB_OUTPUT + echo "Running on request" + # using single brackets here to avoid 08 being interpreted as octal + # https://github.com/carpentries/sandpaper/issues/250 + elif [ `date +%d` -le 7 ]; then + # If the Tuesday lands in the first week of the month, run it + echo "ok=true" >> $GITHUB_OUTPUT + echo "Running on schedule" + else + echo "ok=false" >> $GITHUB_OUTPUT + echo "Not Running Today" + fi + + check_renv: + name: "Check if We Need {renv}" + runs-on: ubuntu-latest + needs: preflight + if: ${{ needs.preflight.outputs.ok == 'true'}} + outputs: + needed: ${{ steps.renv.outputs.exists }} + steps: + - name: "Checkout Lesson" + uses: actions/checkout@v4 + - id: renv + run: | + if [[ -d renv ]]; then + echo "exists=true" >> $GITHUB_OUTPUT + fi + + check_token: + name: "Check SANDPAPER_WORKFLOW token" + runs-on: ubuntu-latest + needs: check_renv + if: ${{ needs.check_renv.outputs.needed == 'true' }} + outputs: + workflow: ${{ steps.validate.outputs.wf }} + repo: ${{ steps.validate.outputs.repo }} + steps: + - name: "validate token" + id: validate + uses: carpentries/actions/check-valid-credentials@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + + update_cache: + name: "Update Package Cache" + needs: check_token + if: ${{ needs.check_token.outputs.repo== 'true' }} + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + steps: + + - name: "Checkout Lesson" + uses: actions/checkout@v4 + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Update {renv} deps and determine if a PR is needed" + id: update + uses: carpentries/actions/update-lockfile@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: Create Pull Request + id: cpr + if: ${{ steps.update.outputs.n > 0 }} + uses: carpentries/create-pull-request@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + delete-branch: true + branch: "update/packages" + commit-message: "[actions] update ${{ steps.update.outputs.n }} packages" + title: "Update ${{ steps.update.outputs.n }} packages" + body: | + :robot: This is an automated build + + This will update ${{ steps.update.outputs.n }} packages in your lesson with the following versions: + + ``` + ${{ steps.update.outputs.report }} + ``` + + :stopwatch: In a few minutes, a comment will appear that will show you how the output has changed based on these updates. + + If you want to inspect these changes locally, you can use the following code to check out a new branch: + + ```bash + git fetch origin update/packages + git checkout update/packages + ``` + + - Auto-generated by [create-pull-request][1] on ${{ steps.update.outputs.date }} + + [1]: https://github.com/carpentries/create-pull-request/tree/main + labels: "type: package cache" + draft: false diff --git a/.github/workflows/update-workflows.yaml b/.github/workflows/update-workflows.yaml new file mode 100644 index 0000000..a2d6fee --- /dev/null +++ b/.github/workflows/update-workflows.yaml @@ -0,0 +1,66 @@ +name: "02 Maintain: Update Workflow Files" + +on: + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build (enter github username to tag yourself)?' + required: true + default: 'weekly run' + clean: + description: 'Workflow files/file extensions to clean (no wildcards, enter "" for none)' + required: false + default: '.yaml' + schedule: + # Run every Tuesday + - cron: '0 0 * * 2' + +jobs: + check_token: + name: "Check SANDPAPER_WORKFLOW token" + runs-on: ubuntu-latest + outputs: + workflow: ${{ steps.validate.outputs.wf }} + repo: ${{ steps.validate.outputs.repo }} + steps: + - name: "validate token" + id: validate + uses: carpentries/actions/check-valid-credentials@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + + update_workflow: + name: "Update Workflow" + runs-on: ubuntu-latest + needs: check_token + if: ${{ needs.check_token.outputs.workflow == 'true' }} + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: Update Workflows + id: update + uses: carpentries/actions/update-workflows@main + with: + clean: ${{ github.event.inputs.clean }} + + - name: Create Pull Request + id: cpr + if: "${{ steps.update.outputs.new }}" + uses: carpentries/create-pull-request@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + delete-branch: true + branch: "update/workflows" + commit-message: "[actions] update sandpaper workflow to version ${{ steps.update.outputs.new }}" + title: "Update Workflows to Version ${{ steps.update.outputs.new }}" + body: | + :robot: This is an automated build + + Update Workflows from sandpaper version ${{ steps.update.outputs.old }} -> ${{ steps.update.outputs.new }} + + - Auto-generated by [create-pull-request][1] on ${{ steps.update.outputs.date }} + + [1]: https://github.com/carpentries/create-pull-request/tree/main + labels: "type: template and tools" + draft: false diff --git a/.gitignore b/.gitignore index b8b053f..bd87454 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,51 @@ -*.pyc -*~ -.DS_Store -.ipynb_checkpoints -.jekyll-cache -.sass-cache -__pycache__ -_site -files/*.gtf -files/*.fb -files/*.fastq -files/*.tsv -Gemfile.lock -scratch/ -.vendor +# sandpaper files +episodes/*html +site/* +!site/README.md + +# History files +.Rhistory +.Rapp.history + +# Session Data files +.RData + +# User-specific files +.Ruserdata + +# Example code in package build process +*-Ex.R + +# Output files from R CMD build +/*.tar.gz + +# Output files from R CMD check +/*.Rcheck/ + +# RStudio files +.Rproj.user/ + +# produced vignettes +vignettes/*.html +vignettes/*.pdf + +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 +.httr-oauth + +# knitr and R markdown default cache directories +*_cache/ +/cache/ + +# Temporary files created by R markdown +*.utf8.md +*.knit.md + +# R Environment Variables +.Renviron + +# pkgdown site +docs/ + +# translation temp files +po/*~ + diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..a2c8a66 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,22 @@ +# This template CITATION.cff file was generated with cffinit. +# Visit https://bit.ly/cffinit to replace its contents +# with information about your lesson. +# Remember to update this file periodically, +# ensuring that the author list and other fields remain accurate. + +cff-version: 1.2.0 +title: FIXME +message: >- + Please cite this lesson using the information in this file + when you refer to it in publications, and/or if you + re-use, adapt, or expand on the content in your own + training material. +type: dataset +authors: + - given-names: FIXME + family-names: FIXME +abstract: >- + FIXME Replace this with a short abstract describing the + lesson, e.g. its target audience and main intended + learning objectives. +license: CC-BY-4.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c3b9669..f19b804 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,11 +1,13 @@ --- -layout: page title: "Contributor Code of Conduct" --- + As contributors and maintainers of this project, -we pledge to follow the [Carpentry Code of Conduct][coc]. +we pledge to follow the [The Carpentries Code of Conduct][coc]. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by following our [reporting guidelines][coc-reporting]. -{% include links.md %} + +[coc-reporting]: https://docs.carpentries.org/topic_folders/policies/incident-reporting.html +[coc]: https://docs.carpentries.org/topic_folders/policies/code-of-conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bb18b7..6c2b81c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,70 +1,65 @@ -# Contributing +## Contributing -[Software Carpentry][swc-site] and [Data Carpentry][dc-site] are open source +[The Carpentries][cp-site] ([Software Carpentry][swc-site], [Data +Carpentry][dc-site], and [Library Carpentry][lc-site]) are open source projects, and we welcome contributions of all kinds: new lessons, fixes to existing material, bug reports, and reviews of proposed changes are all welcome. -## Contributor Agreement +### Contributor Agreement By contributing, you agree that we may redistribute your work under [our license](LICENSE.md). In exchange, we will address your issues and/or assess your change proposal as promptly as we can, and help you become a member of our -community. Everyone involved in [Software Carpentry][swc-site] and [Data -Carpentry][dc-site] agrees to abide by our [code of -conduct](CODE_OF_CONDUCT.md). +community. Everyone involved in [The Carpentries][cp-site] agrees to abide by +our [code of conduct](CODE_OF_CONDUCT.md). -## How to Contribute +### How to Contribute The easiest way to get started is to file an issue to tell us about a spelling mistake, some awkward wording, or a factual error. This is a good way to introduce yourself and to meet some of our community members. 1. If you do not have a [GitHub][github] account, you can [send us comments by - email][email]. However, we will be able to respond more quickly if you use + email][contact]. However, we will be able to respond more quickly if you use one of the other methods described below. + 2. If you have a [GitHub][github] account, or are willing to [create one][github-join], but do not know how to use Git, you can report problems - or suggest improvements by [creating an issue][issues]. This allows us to - assign the item to someone and to respond to it in a threaded discussion. + or suggest improvements by [creating an issue][repo-issues]. This allows us + to assign the item to someone and to respond to it in a threaded discussion. + 3. If you are comfortable with Git, and would like to add or change material, you can submit a pull request (PR). Instructions for doing this are - [included below](#using-github). - -## Where to Contribute - -1. If you wish to change this lesson, please work in - , which can be viewed at - . -2. If you wish to change the example lesson, please work in - , which documents the format - of our lessons and can be viewed at - . -3. If you wish to change the template used for workshop websites, please work - in . The home page of that - repository explains how to set up workshop websites, while the extra pages - in provide more background - on our design choices. -4. If you wish to change CSS style files, tools, or HTML boilerplate for - lessons or workshops stored in `_includes` or `_layouts`, please work in - . - -## What to Contribute + [included below](#using-github). For inspiration about changes that need to + be made, check out the [list of open issues][issues] across the Carpentries. + +Note: if you want to build the website locally, please refer to [The Workbench +documentation][template-doc]. + +### Where to Contribute + +1. If you wish to change this lesson, add issues and pull requests here. +2. If you wish to change the template used for workshop websites, please refer + to [The Workbench documentation][template-doc]. + + +### What to Contribute There are many ways to contribute, from writing new exercises and improving existing ones to updating or filling in the documentation and submitting [bug -reports][issues] about things that don't work, aren't clear, or are missing. If -you are looking for ideas, please see the 'Issues' tab for a list of issues -associated with this repository, or you may also look at the issues for [Data -Carpentry][dc-issues] and [Software Carpentry][swc-issues] projects. +reports][issues] about things that do not work, are not clear, or are missing. +If you are looking for ideas, please see [the list of issues for this +repository][repo-issues], or the issues for [Data Carpentry][dc-issues], +[Library Carpentry][lc-issues], and [Software Carpentry][swc-issues] projects. Comments on issues and reviews of pull requests are just as welcome: we are -smarter together than we are on our own. Reviews from novices and newcomers are -particularly valuable: it's easy for people who have been using these lessons -for a while to forget how impenetrable some of this material can be, so fresh -eyes are always welcome. +smarter together than we are on our own. **Reviews from novices and newcomers +are particularly valuable**: it's easy for people who have been using these +lessons for a while to forget how impenetrable some of this material can be, so +fresh eyes are always welcome. -## What *Not* to Contribute +### What *Not* to Contribute Our lessons already contain more material than we can cover in a typical workshop, so we are usually *not* looking for more concepts or tools to add to @@ -78,50 +73,51 @@ platform. Our workshops typically contain a mixture of Windows, macOS, and Linux users; in order to be usable, our lessons must run equally well on all three. -## Using GitHub +### Using GitHub If you choose to contribute via GitHub, you may want to look at [How to -Contribute to an Open Source Project on GitHub][how-contribute]. To manage -changes, we follow [GitHub flow][github-flow]. Each lesson has two maintainers -who review issues and pull requests or encourage others to do so. The -maintainers are community volunteers and have final say over what gets merged -into the lesson. To use the web interface for contributing to a lesson: - -1. Fork the originating repository to your GitHub profile. -2. Within your version of the forked repository, move to the `gh-pages` branch - and create a new branch for each significant change being made. -3. Navigate to the file(s) you wish to change within the new branches and make - revisions as required. -4. Commit all changed files within the appropriate branches. -5. Create individual pull requests from each of your changed branches to the - `gh-pages` branch within the originating repository. -6. If you receive feedback, make changes using your issue-specific branches of - the forked repository and the pull requests will update automatically. -7. Repeat as needed until all feedback has been addressed. - -When starting work, please make sure your clone of the originating `gh-pages` -branch is up-to-date before creating your own revision-specific branch(es) from -there. Additionally, please only work from your newly-created branch(es) and -*not* your clone of the originating `gh-pages` branch. Lastly, published copies -of all the lessons are available in the `gh-pages` branch of the originating -repository for reference while revising. - -## Other Resources - -General discussion of [Software Carpentry][swc-site] and [Data -Carpentry][dc-site] happens on the [discussion mailing list][discuss-list], -which everyone is welcome to join. You can also [reach us by email][email]. - -[email]: mailto:admin@software-carpentry.org +Contribute to an Open Source Project on GitHub][how-contribute]. In brief, we +use [GitHub flow][github-flow] to manage changes: + +1. Create a new branch in your desktop copy of this repository for each + significant change. +2. Commit the change in that branch. +3. Push that branch to your fork of this repository on GitHub. +4. Submit a pull request from that branch to the [upstream repository][repo]. +5. If you receive feedback, make changes on your desktop and push to your + branch on GitHub: the pull request will update automatically. + +NB: The published copy of the lesson is usually in the `main` branch. + +Each lesson has a team of maintainers who review issues and pull requests or +encourage others to do so. The maintainers are community volunteers, and have +final say over what gets merged into the lesson. + +### Other Resources + +The Carpentries is a global organisation with volunteers and learners all over +the world. We share values of inclusivity and a passion for sharing knowledge, +teaching and learning. There are several ways to connect with The Carpentries +community listed at including via social +media, slack, newsletters, and email lists. You can also [reach us by +email][contact]. + +[repo]: https://example.com/FIXME +[repo-issues]: https://example.com/FIXME/issues +[contact]: mailto:team@carpentries.org +[cp-site]: https://carpentries.org/ [dc-issues]: https://github.com/issues?q=user%3Adatacarpentry -[dc-lessons]: http://datacarpentry.org/lessons/ -[dc-site]: http://datacarpentry.org/ -[discuss-list]: http://lists.software-carpentry.org/listinfo/discuss +[dc-lessons]: https://datacarpentry.org/lessons/ +[dc-site]: https://datacarpentry.org/ +[discuss-list]: https://carpentries.topicbox.com/groups/discuss [github]: https://github.com [github-flow]: https://guides.github.com/introduction/flow/ [github-join]: https://github.com/join -[how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github -[issues]: https://guides.github.com/features/issues/ +[how-contribute]: https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github +[issues]: https://carpentries.org/help-wanted-issues/ +[lc-issues]: https://github.com/issues?q=user%3ALibraryCarpentry [swc-issues]: https://github.com/issues?q=user%3Aswcarpentry [swc-lessons]: https://software-carpentry.org/lessons/ [swc-site]: https://software-carpentry.org/ +[lc-site]: https://librarycarpentry.org/ +[template-doc]: https://carpentries.github.io/workbench/ diff --git a/FIXME.Rproj b/FIXME.Rproj new file mode 100644 index 0000000..718d273 --- /dev/null +++ b/FIXME.Rproj @@ -0,0 +1,19 @@ +Version: 1.0 + +RestoreWorkspace: No +SaveWorkspace: No +AlwaysSaveHistory: Default + +EnableCodeIndexing: Yes +UseSpacesForTab: Yes +NumSpacesForTab: 2 +Encoding: UTF-8 + +RnwWeave: Sweave +LaTeX: pdfLaTeX + +AutoAppendNewline: Yes +StripTrailingWhitespace: Yes +LineEndingConversion: Posix + +BuildType: Website diff --git a/LICENSE.md b/LICENSE.md index 179758a..7632871 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,83 +1,79 @@ --- -layout: page title: "Licenses" -permalink: /license/ --- + ## Instructional Material -All Software Carpentry and Data Carpentry instructional material is -made available under the [Creative Commons Attribution -license][cc-by-human]. The following is a human-readable summary of +All Carpentries (Software Carpentry, Data Carpentry, and Library Carpentry) +instructional material is made available under the [Creative Commons +Attribution license][cc-by-human]. The following is a human-readable summary of (and not a substitute for) the [full legal text of the CC BY 4.0 license][cc-by-legal]. You are free: -* to **Share**---copy and redistribute the material in any medium or format -* to **Adapt**---remix, transform, and build upon the material +- to **Share**---copy and redistribute the material in any medium or format +- to **Adapt**---remix, transform, and build upon the material for any purpose, even commercially. -The licensor cannot revoke these freedoms as long as you follow the -license terms. +The licensor cannot revoke these freedoms as long as you follow the license +terms. Under the following terms: -* **Attribution**---You must give appropriate credit (mentioning that - your work is derived from work that is Copyright © Software - Carpentry and, where practical, linking to - http://software-carpentry.org/), provide a [link to the - license][cc-by-human], and indicate if changes were made. You may do - so in any reasonable manner, but not in any way that suggests the - licensor endorses you or your use. +- **Attribution**---You must give appropriate credit (mentioning that your work + is derived from work that is Copyright (c) The Carpentries and, where + practical, linking to ), provide a [link to the + license][cc-by-human], and indicate if changes were made. You may do so in + any reasonable manner, but not in any way that suggests the licensor endorses + you or your use. -**No additional restrictions**---You may not apply legal terms or -technological measures that legally restrict others from doing -anything the license permits. With the understanding that: +- **No additional restrictions**---You may not apply legal terms or + technological measures that legally restrict others from doing anything the + license permits. With the understanding that: Notices: -* You do not have to comply with the license for elements of the - material in the public domain or where your use is permitted by an - applicable exception or limitation. -* No warranties are given. The license may not give you all of the - permissions necessary for your intended use. For example, other - rights such as publicity, privacy, or moral rights may limit how you - use the material. +* You do not have to comply with the license for elements of the material in + the public domain or where your use is permitted by an applicable exception + or limitation. +* No warranties are given. The license may not give you all of the permissions + necessary for your intended use. For example, other rights such as publicity, + privacy, or moral rights may limit how you use the material. ## Software -Except where otherwise noted, the example programs and other software -provided by Software Carpentry and Data Carpentry are made available under the -[OSI][osi]-approved -[MIT license][mit-license]. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Except where otherwise noted, the example programs and other software provided +by The Carpentries are made available under the [OSI][osi]-approved [MIT +license][mit-license]. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. ## Trademark -"Software Carpentry" and "Data Carpentry" and their respective logos -are registered trademarks of [NumFOCUS][numfocus]. +"The Carpentries", "Software Carpentry", "Data Carpentry", and "Library +Carpentry" and their respective logos are registered trademarks of [Community +Initiatives][ci]. [cc-by-human]: https://creativecommons.org/licenses/by/4.0/ [cc-by-legal]: https://creativecommons.org/licenses/by/4.0/legalcode -[mit-license]: http://opensource.org/licenses/mit-license.html -[numfocus]: http://numfocus.org/ -[osi]: http://opensource.org +[mit-license]: https://opensource.org/licenses/mit-license.html +[ci]: https://communityin.org/ +[osi]: https://opensource.org diff --git a/_config.yml b/_config.yml deleted file mode 100644 index 40d5f87..0000000 --- a/_config.yml +++ /dev/null @@ -1,104 +0,0 @@ -#------------------------------------------------------------ -# Values for this lesson. -#------------------------------------------------------------ - -# Which carpentry is this ("swc", "dc", or "lc")? -carpentry: "swc" - -# Overall title for pages. -title: "Introduction to High-Performance Computing in Chapel" - -#------------------------------------------------------------ -# Generic settings (should not need to change). -#------------------------------------------------------------ - -# What kind of thing is this ("workshop" or "lesson")? -kind: "lesson" - -# Magic to make URLs resolve both locally and on GitHub. -# See https://help.github.com/articles/repository-metadata-on-github-pages/. -repository: / - -# Email address, no mailto: -email: "team@carpentries.org" - -# Sites. -amy_site: "https://amy.software-carpentry.org/workshops" -dc_site: "http://datacarpentry.org" -swc_github: "https://github.com/swcarpentry" -swc_site: "https://software-carpentry.org" -swc_pages: "https://swcarpentry.github.io" -lc_site: "http://librarycarpentry.github.io/" -template_repo: "https://github.com/swcarpentry/styles" -example_repo: "https://github.com/swcarpentry/lesson-example" -example_site: "https://swcarpentry.github.com/lesson-example" -workshop_repo: "https://github.com/swcarpentry/workshop-template" -workshop_site: "https://swcarpentry.github.io/workshop-template" -training_site: "https://swcarpentry.github.io/instructor-training" - -# Surveys. -pre_survey: "https://www.surveymonkey.com/r/swc_pre_workshop_v1?workshop_id=" -post_survey: "https://www.surveymonkey.com/r/swc_post_workshop_v1?workshop_id=" -training_post_survey: "https://www.surveymonkey.com/r/post-instructor-training" - -# Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). -start_time: 0 - -# Specify that things in the Episodes and Extras collections should be output. -collections: - episodes: - output: true - permalink: /:path/index.html - extras: - output: true - permalink: /:path/index.html - -# Set the default layout for things in the episodes collection. -defaults: - - values: - root: . - layout: page - - scope: - path: "" - type: episodes - values: - root: .. - layout: episode - - scope: - path: "" - type: extras - values: - root: .. - layout: page - -# Files and directories that are not to be copied. -exclude: - - Makefile - - bin/ - - .Rproj.user/ - - .vendor/ - - .docker-vendor/ - -# Turn on built-in syntax highlighting. -highlighter: rouge - -# Invoke the Carpentries theme rather than copying into our repository. -remote_theme: carpentries/carpentries-theme - -# Manual episode numbering -episode_order: - - 01-intro - - 02-variables - - 03-ranges-arrays - - 04-conditionals - - 05-loops - - 06-procedures - - 07-commandargs - - 08-timing - - 11-parallel-intro - - 12-fire-forget-tasks - - 13-synchronization - - 14-parallel-case-study - - 21-locales - - 22-domains - diff --git a/_episodes/01-intro.md b/_episodes/01-intro.md deleted file mode 100644 index d0c1bd8..0000000 --- a/_episodes/01-intro.md +++ /dev/null @@ -1,164 +0,0 @@ ---- -title: "Introduction to Chapel" -teaching: 15 -exercises: 15 -questions: -- "What is Chapel and why is it useful?" -objectives: -- "Write and execute our first chapel program." -keypoints: -- "Chapel is a compiled language - any programs we make must be compiled with `chpl`." -- "The `--fast` flag instructs the Chapel compiler to optimise our code." -- "The `-o` flag tells the compiler what to name our output (otherwise it gets named `a.out`)" ---- - -**_Chapel_** is a modern programming language, developed by _Cray Inc._, that -supports HPC via high-level abstractions for data parallelism and task -parallelism. These abstractions allow the users to express parallel codes in a -natural, almost intuitive, manner. In contrast with other high-level parallel -languages, however, Chapel was designed around a _multi-resolution_ philosophy. -This means that users can incrementally add more detail to their original code -prototype, to optimise it to a particular computer as closely as required. - -In a nutshell, with Chapel we can write parallel code with the simplicity and -readability of scripting languages such as Python or MATLAB, but achieving -performance comparable to compiled languages like C or Fortran (+ traditional -parallel libraries such as MPI or OpenMP). - -In this lesson we will learn the basic elements and syntax of the language; -then we will study **_task parallelism_**, the first level of parallelism in -Chapel, and finally we will use parallel data structures and **_data -parallelism_**, which is the higher level of abstraction, in parallel -programming, offered by Chapel. - -## Getting started - -Chapel is a compilable language which means that we must **_compile_** our -**_source code_** to generate a **_binary_** or **_executable_** that we can -then run in the computer. - -Chapel source code must be written in text files with the extension -**_.chpl_**. Let's write a simple "hello world"-type program to demonstrate how -we write Chapel code! Using your favourite text editor, create the file -`hello.chpl` with the following content: - -``` -writeln('If we can see this, everything works!'); -``` -{: .bash} - -This program can then be compiled with the following bash command: - -~~~ -chpl --fast hello.chpl -o hello.o -~~~ -{: .bash} - -The flag `--fast` indicates the compiler to optimise the binary to run as fast -as possible in the given architecture. The `-o` option tells Chapel what to -call the final output program, in this case `hello.o`. - -To run the code, you execute it as you would any other program: - -~~~ -./hello.o -~~~ -{: .bash} -``` -If we can see this, everything works! -``` -{: .output} - -## Running on a cluster - -Depending on the code, it might utilise several or even all cores on the -current node. The command above implies that you are allowed to utilise all -cores. This might not be the case on an HPC cluster, where a login node is -shared by many people at the same time, and where it might not be a good idea -to occupy all cores on a login node with CPU-intensive tasks. - -On Compute Canada clusters Cedar and Graham we have two versions of Chapel, one -is a single-locale (single-node) Chapel, and the other is a multi-locale -(multi-node) Chapel. For now, we will start with single-locale Chapel. If you -are logged into Cedar or Graham, you'll need to load the single-locale Chapel -module: - -~~~ -module load gcc chapel-multicore -~~~ -{: .bash} - -Then, for running a test code on a cluster you would submit an interactive job -to the queue - -~~~ -salloc --time=0:30:0 --ntasks=1 --cpus-per-task=3 --mem-per-cpu=1000 --account=def-guest -~~~ -{: .bash} - -and then inside that job compile and run the test code - -~~~ -chpl --fast hello.chpl -o hello.o -./hello.o -~~~ -{: .bash} - -For production jobs, you would compile the code and then submit a batch script -to the queue: - -~~~ -chpl --fast hello.chpl -o hello.o -sbatch script.sh -~~~ -{: .bash} - -where the script `script.sh` would set all Slurm variables and call the -executable `mybinary`. - -## Case study - -Along all the Chapel lessons we will be using the following _case study_ as the -leading thread of the discussion. Essentially, we will be building, step by -step, a Chapel code to solve the **_Heat transfer_** problem described below. -Then we will parallelize the code to improve its performance. - -Suppose that we have a square metallic plate with some initial heat -distribution or **_initial conditions_**. We want to simulate the evolution of -the temperature across the plate when its border is in contact with a different -heat distribution that we call the **_boundary conditions_**. - -The Laplace equation is the mathematical model for the evolution of the -temperature in the plate. To solve this equation numerically, we need to -**_discretise_** it, i.e. to consider the plate as a grid, or matrix of points, -and to evaluate the temperature on each point at each iteration, according to -the following **_difference equation_**: - -``` -T[i,j] = 0.25 (Tp[i-1,j] + Tp[i+1,j] + Tp[i,j-1] + Tp[i,j+1]) -``` -{: .source} - -Here `T` stands for the temperature at the current iteration, while `Tp` -contains the temperature calculated at the past iteration (or the initial -conditions in case we are at the first iteration). The indices `i` and `j` -indicate that we are working on the point of the grid located at the *i*th row -and the *j*th column. - -So, our objective is to: - -> ## Goals -> 1. Write a code to implement the difference equation above.The code should -> have the following requirements: -> -> * It should work for any given number of rows and columns in the grid. -> * It should run for a given number of iterations, or until the difference -> between `T` and `Tp` is smaller than a given tolerance value. -> * It should output the temperature at a desired position on the grid every -> given number of iterations. -> -> 2. Use task parallelism to improve the performance of the code and run it in -> the cluster -> 3. Use data parallelism to improve the performance of the code and run it in -> the cluster. -{: .checklist} diff --git a/_episodes/05-loops.md b/_episodes/05-loops.md deleted file mode 100644 index 4a4e808..0000000 --- a/_episodes/05-loops.md +++ /dev/null @@ -1,335 +0,0 @@ ---- -title: "Getting started with loops" -teaching: 60 -exercises: 30 -questions: -- "How do I get run the same piece of code repeatedly?" -objectives: -- "First objective." -keypoints: -- "Use `for` statement to organise a loop." ---- - -To compute the current temperature of an element of `temp`, we need to add all -the surrounding elements in `past_temp`, and divide the result by 4. And, -essentially, we need to repeat this process for all the elements of `temp`, or, -in other words, we need to *iterate* over the elements of `temp`. When it comes -to iterate over a given number of elements, the **_for-loop_** is what we want -to use. The for-loop has the following general syntax: - -``` -for index in iterand do -{instructions} -``` -{: .source} - -The *iterand* is a function or statement that expresses an iteration; it could -be the range 1..15, for example. *index* is a variable that exists only in the -context of the for-loop, and that will be taking the different values yielded -by the iterand. The code flows as follows: index takes the first value yielded -by the iterand, and keeps it until all the instructions inside the curly -brackets are executed one by one; then, index takes the second value yielded by -the iterand, and keeps it until all the instructions are executed again. This -pattern is repeated until index takes all the different values expressed by the -iterand. - -This for loop, for example - -~~~ -//calculate the new current temperatures (temp) using the past temperatures (past_temp) -for i in 1..rows do -{ - //do this for every row -} -~~~ -{: .source} - -will allow us to iterate over the rows of `temp`. Now, for each row we also -need to iterate over all the columns in order to access every single element of -`temp`. This can be done with nested for loops like this - -~~~ -//calculate the new current temperatures (temp) using the past temperatures (past_temp) -for i in 1..rows do -{ - //do this for every row - for j in 1..cols do - { - //and this for every column in the row i - } -} -~~~ -{: .source} - -Now, inside the inner loop, we can use the indices `i` and `j` to perform the -required computations as follows: - -~~~ -//calculate the new current temperatures (temp) using the past temperatures (past_temp) -for i in 1..rows do -{ - //do this for every row - for j in 1..cols do - { - //and this for every column in the row i - temp[i,j]=(past_temp[i-1,j]+past_temp[i+1,j]+past_temp[i,j-1]+past_temp[i,j+1])/4; - } -} -past_temp=temp; -~~~ -{: .source} - -Note that at the end of the outer for-loop, when all the elements in `temp` are -already calculated, we update `past_temp` with the values of `temp`; this way -everything is set up for the next iteration of the main while statement. - -Now let's compile and execute our code again: - -~~~ ->> chpl base_solution.chpl -o base_solution ->> ./base_solution -~~~ -{: .bash} - -~~~ -The simulation will consider a matrix of 100 by 100 elements, -it will run up to 500 iterations, or until the largest difference -in temperature between iterations is less than 0.0001. -You are interested in the evolution of the temperature at the -position (50,50) of the matrix... - -and here we go... -Temperature at iteration 0: 25.0 -Temperature at iteration 20: 25.0 -Temperature at iteration 40: 25.0 -Temperature at iteration 60: 25.0 -Temperature at iteration 80: 25.0 -Temperature at iteration 100: 25.0 -Temperature at iteration 120: 25.0 -Temperature at iteration 140: 25.0 -Temperature at iteration 160: 25.0 -Temperature at iteration 180: 25.0 -Temperature at iteration 200: 25.0 -Temperature at iteration 220: 24.9999 -Temperature at iteration 240: 24.9996 -Temperature at iteration 260: 24.9991 -Temperature at iteration 280: 24.9981 -Temperature at iteration 300: 24.9963 -Temperature at iteration 320: 24.9935 -Temperature at iteration 340: 24.9893 -Temperature at iteration 360: 24.9833 -Temperature at iteration 380: 24.9752 -Temperature at iteration 400: 24.9644 -Temperature at iteration 420: 24.9507 -Temperature at iteration 440: 24.9337 -Temperature at iteration 460: 24.913 -Temperature at iteration 480: 24.8883 -Temperature at iteration 500: 24.8595 -~~~ -{: .output} - -As we can see, the temperature in the middle of the plate (position 50,50) is -slowly decreasing as the plate is cooling down. - -> ## Exercise 1 -> -> What would be the temperature at the top right corner of the plate? The -> border of the plate is in contact with the boundary conditions, which are set -> to zero, so we expect the temperature at these points to decrease faster. -> Modify the code to see the temperature at the top right corner. -> -> > ## Solution -> > To see the evolution of the temperature at the top right corner of the plate, we just need to modify `x` and `y`. This corner correspond to the first row (`x=1`) and the last column (`y=cols`) of the plate. -> > ~~~ -> > >> chpl base_solution.chpl -o base_solution -> > >> ./base_solution -> > ~~~ -> > {: .bash} -> > ~~~ -> > The simulation will consider a matrix of 100 by 100 elements, -> > it will run up to 500 iterations, or until the largest difference -> > in temperature between iterations is less than 0.0001. -> > You are interested in the evolution of the temperature at the position (1,100) of the matrix... -> > -> > and here we go... -> > Temperature at iteration 0: 25.0 -> > Temperature at iteration 20: 1.48171 -> > Temperature at iteration 40: 0.767179 -> > ... -> > Temperature at iteration 460: 0.068973 -> > Temperature at iteration 480: 0.0661081 -> > Temperature at iteration 500: 0.0634717 -> > ~~~ -> > {: .output} -> {: .solution} -{: .challenge} - -> ## Exercise 2 -> Now let's have some more interesting boundary conditions. Suppose that the -> plate is heated by a source of 80 degrees located at the bottom right corner, -> and that the temperature on the rest of the border decreases linearly as one -> gets farther form the corner (see the image below). Utilise for loops to -> setup the described boundary conditions. Compile and run your code to see how -> the temperature is changing now. -> -> > ## Solution -> > -> > To get the linear distribution, the 80 degrees must be divided by the number -> > of rows or columns in our plate. So, the following couple of for loops will -> > give us what we want; -> > ~~~ -> > //this setup the boundary conditions -> > for i in 1..rows do -> > { -> > past_temp[i,cols+1]=i*80.0/rows; -> > temp[i,cols+1]=i*80.0/rows; -> > } -> > for j in 1..cols do -> > { -> > past_temp[rows+1,j]=j*80.0/cols; -> > temp[rows+1,j]=j*80.0/cols; -> > } -> > ~~~ -> > {: .source} -> > -> > Note that the boundary conditions must be set in both arrays, `past_temp` -> > and `temp`, otherwise, they will be set to zero again after the first -> > iteration. Also note that 80 degrees are written as a real numbe r 80.0. -> > The division of integers in Chapel returns an integer, then, as `rows` -> > and `cols` are integers, we must have 80 as real so that the quotient is -> > not truncated. -> > ~~~ -> > >> chpl base_solution.chpl -o base_solution -> > >> ./base_solution -> > ~~~ -> > {: .bash} -> > ~~~ -> > -> > The simulation will consider a matrix of 100 by 100 elements, it will run -> > up to 500 iterations, or until the largest difference in temperature -> > between iterations is less than 0.0001. You are interested in the evolution -> > of the temperature at the position (1,100) of the matrix... -> > -> > and here we go... -> > Temperature at iteration 0: 25.0 -> > Temperature at iteration 20: 2.0859 -> > Temperature at iteration 40: 1.42663 -> > ... -> > Temperature at iteration 460: 0.826941 -> > Temperature at iteration 480: 0.824959 -> > Temperature at iteration 500: 0.823152 -> > ~~~ -> > {: .output} -> {: .solution} -{: .challenge} - -> ## Exercise 3 -> -> So far, `curdif` has been always equal to `mindif`, which means that our main -> while loop will always run the 500 iterations. So let's update `curdif` after -> each iteration. Use what we have studied so far to write the required piece -> of code. -> > ## Solution -> > -> > The idea is simple, after each iteration of the while loop, we must compare -> > all elements of `temp` and `past_temp`, find the greatest difference, and -> > update `curdif` with that value. The next nested for loops do the job: -> > -> > ~~~ -> > //update curdif, the greatest difference between temp and past_temp -> > curdif=0; -> > for i in 1..rows do -> > { -> > for j in 1..cols do -> > { -> > tt=abs(temp[i,j]-past_temp[i,j]); -> > if tt>curdif then curdif=tt; -> > } -> > } -> > ~~~ -> > {: .source} -> > -> > Clearly there is no need to keep the difference at every single position in -> > the array, we just need to update `curdif` if we find a greater one. -> > ~~~ -> > >> chpl base_solution.chpl -o base_solution -> > >> ./base_solution -> > ~~~ -> > {: .bash} -> > ~~~ -> > The simulation will consider a matrix of 100 by 100 elements, -> > it will run up to 500 iterations, or until the largest difference -> > in temperature between iterations is less than 0.0001. -> > You are interested in the evolution of the temperature at the -> > position (1,100) of the matrix... -> > -> > and here we go... -> > Temperature at iteration 0: 25.0 -> > Temperature at iteration 20: 2.0859 -> > Temperature at iteration 40: 1.42663 -> > ... -> > Temperature at iteration 460: 0.826941 -> > Temperature at iteration 480: 0.824959 -> > Temperature at iteration 500: 0.823152 -> > ~~~ -> > {: .output} -> {: .solution} -{: .challenge} - -Now, after Exercise 3 we should have a working program to simulate our heat -transfer equation. Let's just print some additional useful information, - -~~~ -//print final information -writeln('\nFinal temperature at the desired position after ',c,' iterations is: ',temp[x,y]); -writeln('The difference in temperatures between the last two iterations was: ',curdif,'\n'); -~~~ -{: .source} - -and compile and execute our final code, - -~~~ ->> chpl base_solution.chpl -o base_solution ->> ./base_solution -~~~ -{: .bash} - -~~~ -The simulation will consider a matrix of 100 by 100 elements, -it will run up to 500 iterations, or until the largest difference -in temperature between iterations is less than 0.0001. -You are interested in the evolution of the temperature at the -position (1,100) of the matrix... - -and here we go... -Temperature at iteration 0: 25.0 -Temperature at iteration 20: 2.0859 -Temperature at iteration 40: 1.42663 -Temperature at iteration 60: 1.20229 -Temperature at iteration 80: 1.09044 -Temperature at iteration 100: 1.02391 -Temperature at iteration 120: 0.980011 -Temperature at iteration 140: 0.949004 -Temperature at iteration 160: 0.926011 -Temperature at iteration 180: 0.908328 -Temperature at iteration 200: 0.894339 -Temperature at iteration 220: 0.88302 -Temperature at iteration 240: 0.873688 -Temperature at iteration 260: 0.865876 -Temperature at iteration 280: 0.85925 -Temperature at iteration 300: 0.853567 -Temperature at iteration 320: 0.848644 -Temperature at iteration 340: 0.844343 -Temperature at iteration 360: 0.840559 -Temperature at iteration 380: 0.837205 -Temperature at iteration 400: 0.834216 -Temperature at iteration 420: 0.831537 -Temperature at iteration 440: 0.829124 -Temperature at iteration 460: 0.826941 -Temperature at iteration 480: 0.824959 -Temperature at iteration 500: 0.823152 - -Final temperature at the desired position after 500 iterations is: 0.823152 -The greatest difference in temperatures between the last two iterations was: 0.0258874 -~~~ -{: .output} diff --git a/_episodes/07-commandargs.md b/_episodes/07-commandargs.md deleted file mode 100644 index 4aa69a2..0000000 --- a/_episodes/07-commandargs.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -title: "Using command-line arguments" -teaching: 60 -exercises: 30 -questions: -- "How do I use the same program for multiple use-cases?" -objectives: -- "First objective." -keypoints: -- "Config variables accept values from the command line at runtime, without you having to recompile the code." ---- - -From the last run of our code, we can see that 500 iterations is not enough to -get to a _steady state_ (a state where the difference in temperature does not -vary too much, i.e. `curdif`<`mindif`). Now, if we want to change the number of -iterations we would need to modify `niter` in the code, and compile it again. -What if we want to change the number of rows and columns in our grid to have -more precision, or if we want to see the evolution of the temperature at a -different point (x,y)? The answer would be the same, modify the code and -compile it again! - -No need to say that this would be very tedious and inefficient. A better -scenario would be if we can pass the desired configuration values to our binary -when it is called at the command line. The Chapel mechanism for this is to use -**_config_** variables. When a variable is declared with the `config` keyword, -in addition to `var` or `const`, like this: - -~~~ -config const niter = 500; //number of iterations -~~~ -{:.source} - -~~~ ->> chpl base_solution.chpl -o base_solution -~~~ -{:.bash} - -it can be initialised with a specific value, when executing the code at the -command line, using the syntax: - -~~~ ->> ./base_solution --niter=3000 -~~~ -{:.bash} - -~~~ -The simulation will consider a matrix of 100 by 100 elements, -it will run up to 3000 iterations, or until the largest difference -in temperature between iterations is less than 0.0001. -You are interested in the evolution of the temperature at the -position (1,100) of the matrix... - -and here we go... -Temperature at iteration 0: 25.0 -Temperature at iteration 20: 2.0859 -Temperature at iteration 40: 1.42663 -... -Temperature at iteration 2980: 0.793969 -Temperature at iteration 3000: 0.793947 - -Final temperature at the desired position after 3000 iterations is: 0.793947 -The greatest difference in temperatures between the last two iterations was: 0.000350086 -~~~ -{:.output} - -> ## Exercise 4 -> -> Make `n`, `x`, `y`, `mindif`, `rows` and `cols` configurable variables, and -> test the code simulating different configurations. What can you conclude -> about the performance of the code? -> -> > ## Solution -> > -> > For example, lets use a 650 x 650 grid and observe the evolution of the -> > temperature at the position (200,300) for 10000 iterations or until the -> > difference of temperature between iterations is less than 0.002; also, let's -> > print the temperature every 1000 iterations. -> > -> > ~~~ -> > >> ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 -> > ~~~ -> > {:.bash} -> > ~~~ -> > The simulation will consider a matrix of 650 by 650 elements, -> > it will run up to 10000 iterations, or until the largest difference -> > in temperature between iterations is less than 0.002. -> > You are interested in the evolution of the temperature at the position -> > (200,300) of the matrix... -> > -> > and here we go... -> > Temperature at iteration 0: 25.0 -> > Temperature at iteration 1000: 25.0 -> > Temperature at iteration 2000: 25.0 -> > Temperature at iteration 3000: 25.0 -> > Temperature at iteration 4000: 24.9998 -> > Temperature at iteration 5000: 24.9984 -> > Temperature at iteration 6000: 24.9935 -> > Temperature at iteration 7000: 24.9819 -> > -> > Final temperature at the desired position after 7750 iterations is: 24.9671 -> > The greatest difference in temperatures between the last two iterations was: 0.00199985 -> > ~~~ -> > {:.output} -> {:.solution} -{:.challenge} diff --git a/_episodes/11-parallel-intro.md b/_episodes/11-parallel-intro.md deleted file mode 100644 index db9fc39..0000000 --- a/_episodes/11-parallel-intro.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -title: "Intro to parallel computing" -teaching: 60 -exercises: 30 -questions: -- "How does parallel processing work?" -objectives: -- "First objective." -keypoints: -- "Concurrency and locality are orthogonal concepts in Chapel: where the tasks are running may not be indicative of when they run, and you can control both in Chapel." ---- - -The basic concept of parallel computing is simple to understand: we divide our -job into tasks that can be executed at the same time, so that we finish the job -in a fraction of the time that it would have taken if the tasks were executed -one by one. Implementing parallel computations, however, is not always easy, -nor possible... - -Consider the following analogy: - -Suppose that we want to paint the four walls in a room. We'll call this the -*problem*. We can divide our problem into 4 different tasks: paint each of the -walls. In principle, our 4 tasks are independent from each other in the sense -that we don't need to finish one to start one another. We say that we have 4 -**_concurrent tasks_**; the tasks can be executed within the same time frame. -However, this does not mean that the tasks can be executed simultaneously or in -parallel. It all depends on the amount of resources that we have for the tasks. -If there is only one painter, this guy could work for a while in one wall, then -start painting another one, then work for a little bit on the third one, and so -on. **_The tasks are being executed concurrently but not in parallel_**. If we -have two painters for the job, then more parallelism can be introduced. Four -painters could executed the tasks **_truly in parallel_**. - -> ## Key idea -> -> Think of the CPU cores as the painters or workers that will execute your -> concurrent tasks -{:.callout} - -Now imagine that all workers have to obtain their paint from a central -dispenser located at the middle of the room. If each worker is using a -different colour, then they can work **_asynchronously_**, however, if they use -the same colour, and two of them run out of paint at the same time, then they -have to **_synchronise_** to use the dispenser: One must wait while the other -is being serviced. - -> ## Key idea -> -> Think of the shared memory in your computer as the central dispenser for all -> your workers -{:.callout} - -Finally, imagine that we have 4 paint dispensers, one for each worker. In this -scenario, each worker can complete their task totally on their own. They don't -even have to be in the same room, they could be painting walls of different -rooms in the house, in different houses in the city, and different cities in -the country. We need, however, a communication system in place. Suppose that -worker A, for some reason, needs a colour that is only available in the -dispenser of worker B, they must then synchronise: worker A must request the -paint of worker B and worker B must respond by sending the required colour. - -> ## Key idea -> -> Think of the memory distributed on each node of a cluster as the different -> dispensers for your workers -{:.callout} - -A **_fine-grained_** parallel code needs lots of communication or -synchronisation between tasks, in contrast with a **_coarse-grained_** one. An -**_embarrassingly parallel_** problem is one where all tasks can be executed -completely independent from each other (no communications required). - -## Parallel programming in Chapel - -Chapel provides high-level abstractions for parallel programming no matter the -grain size of your tasks, whether they run in a shared memory or a distributed -memory environment, or whether they are executed concurrently or truly in -parallel. As a programmer you can focus in the algorithm: how to divide the -problem into tasks that make sense in the context of the problem, and be sure -that the high-level implementation will run on any hardware configuration. Then -you could consider the details of the specific system you are going to use -(whether it is shared or distributed, the number of cores, etc.) and tune your -code/algorithm to obtain a better performance. - -> ## Key idea -> -> To this effect, **_concurrency_** (the creation and execution of multiple -> tasks), and **_locality_** (in which set of resources these tasks are -> executed) are orthogonal concepts in Chapel. -{:.callout} - -In summary, we can have a set of several tasks; these tasks could be running: - -1. concurrently by the same processor in a single compute node, -2. in parallel by several processors in a single compute node, -3. in parallel by several processors distributed in different compute nodes, or -4. serially (one by one) by several processors distributed in different compute - nodes. - -Similarly, each of these tasks could be using variables located in: - -1. the local memory on the compute node where it is running, or -2. on distributed memory located in other compute nodes. - -And again, Chapel could take care of all the stuff required to run our -algorithm in most of the scenarios, but we can always add more specific detail -to gain performance when targeting a particular scenario. diff --git a/_episodes/12-fire-forget-tasks.md b/_episodes/12-fire-forget-tasks.md deleted file mode 100644 index e4c4f31..0000000 --- a/_episodes/12-fire-forget-tasks.md +++ /dev/null @@ -1,401 +0,0 @@ ---- -title: "Fire-and-forget tasks" -teaching: 60 -exercises: 30 -questions: -- "How do execute work in parallel?" -objectives: -- "First objective." -keypoints: -- "Use `begin` or `cobegin` or `coforall` to spawn new tasks." -- "You can run more than one task per core, as the number of cores on a node is limited." ---- - -A Chapel program always start as a single main thread. You can then start -concurrent tasks with the `begin` statement. A task spawned by the `begin` -statement will run in a different thread while the main thread continues its -normal execution. Consider the following example: - -~~~ -var x=0; - -writeln("This is the main thread starting first task"); -begin -{ - var c=0; - while c<100 - { - c+=1; - writeln('thread 1: ',x+c); - } -} - -writeln("This is the main thread starting second task"); -begin -{ - var c=0; - while c<100 - { - c+=1; - writeln('thread 2: ',x+c); - } -} - -writeln('this is main thread, I am done...'); -~~~ -{: .source} - -~~~ ->> chpl begin_example.chpl -o begin_example ->> ./begin_example -~~~ -{: .bash} - -~~~ -This is the main thread starting first task -This is the main thread starting second task -this is main thread, I am done... -thread 2: 1 -thread 2: 2 -thread 2: 3 -thread 2: 4 -thread 2: 5 -thread 2: 6 -thread 2: 7 -thread 2: 8 -thread 2: 9 -thread 2: 10 -thread 2: 11 -thread 2: 12 -thread 2: 13 -thread 1: 1 -thread 2: 14 -thread 1: 2 -thread 2: 15 -... -thread 2: 99 -thread 1: 97 -thread 2: 100 -thread 1: 98 -thread 1: 99 -thread 1: 100 -~~~ -{: .output} - -As you can see the order of the output is not what we would expected, and -actually it is completely unpredictable. This is a well known effect of -concurrent tasks accessing the same shared resource at the same time (in this -case the screen); the system decides in which order the tasks could write to -the screen. - -> ## Discussion -> -> * What would happen if in the last code we declare `c` in the main thread? -> * What would happen if we try to modify the value of `x`inside a begin -> statement? -> -> Discuss your observations. -> -> > ## Key idea -> > -> > All variables have a **_scope_** in which they can be used. The variables -> > declared inside a concurrent tasks are accessible only by the task. The -> > variables declared in the main task can be read everywhere, but Chapel -> > won't allow two concurrent tasks to try to modify them. -> {: .solution} -{: .discussion} - -> ## Try this... -> -> Are the concurrent tasks, spawned by the last code, running truly in -> parallel? -> -> The answer is: it depends on the number of cores available to your job. To -> verify this, let's modify the code to get the tasks into an infinite loop. -> -> > ~~~ -> > begin -> > { -> > var c=0; -> > while c>-1 -> > { -> > c+=1; -> > //writeln('thread 1: ',x+c); -> > } -> > } -> > ~~~ -> > {: .source} -> > -> > Now submit your job asking for different amount of resources, and use system -> > tools such as `top`or `ps` to monitor the execution of the code. -> > -> > > ## Key idea -> > > -> > > To maximise performance, start as many tasks as cores are available -> > {: .discussion} -> {: .solution} -{: .challenge} - -A slightly more structured way to start concurrent tasks in Chapel is by using -the `cobegin`statement. Here you can start a block of concurrent tasks, one for -each statement inside the curly brackets. The main difference between the -`begin`and `cobegin` statements is that with the `cobegin`, all the spawned -tasks are synchronised at the end of the statement, i.e. the main thread won't -continue its execution until all tasks are done. - -~~~ -var x=0; -writeln("This is the main thread, my value of x is ",x); - -cobegin -{ - { - var x=5; - writeln("this is task 1, my value of x is ",x); - } - writeln("this is task 2, my value of x is ",x); -} - -writeln("this message won't appear until all tasks are done..."); -~~~ -{: .source} - -~~~ ->> chpl cobegin_example.chpl -o cobegin_example ->> ./cobegin_example -~~~ -{: .bash} - -~~~ -This is the main thread, my value of x is 0 -this is task 2, my value of x is 0 -this is task 1, my value of x is 5 -this message won't appear until all tasks are done... -~~~ -{: .output} - -As you may have conclude from the Discussion exercise above, the variables -declared inside a task are accessible only by the task, while those variables -declared in the main task are accessible to all tasks. - -The last, and most useful way to start concurrent/parallel tasks in Chapel, is -the `coforall` loop. This is a combination of the for-loop and the -`cobegin`statements. The general syntax is: - -``` -coforall index in iterand -{instructions} -``` -{: .source} - -This will start a new task, for each iteration. Each tasks will then perform -all the instructions inside the curly brackets. Each task will have a copy of -the variable **_index_** with the corresponding value yielded by the iterand. -This index allows us to _customise_ the set of instructions for each particular -task. - -~~~ -var x=1; -config var numoftasks=2; - -writeln("This is the main task: x = ",x); - -coforall taskid in 1..numoftasks do -{ - var c=taskid+1; - writeln("this is task ",taskid,": x + ",taskid," = ",x+taskid,". My value of c is: ",c); -} - -writeln("this message won't appear until all tasks are done..."); -~~~ -{: .source} - -~~~ -> > chpl coforall_example.chpl -o coforall_example -> > ./coforall_example --numoftasks=5 -~~~ -{: .bash} - -~~~ -This is the main task: x = 1 -this is task 5: x + 5 = 6. My value of c is: 6 -this is task 2: x + 2 = 3. My value of c is: 3 -this is task 4: x + 4 = 5. My value of c is: 5 -this is task 3: x + 3 = 4. My value of c is: 4 -this is task 1: x + 1 = 2. My value of c is: 2 -this message won't appear until all tasks are done... -~~~ -{: .output} - -Notice how we are able to customise the instructions inside the coforall, to -give different results depending on the task that is executing them. Also, -notice how, once again, the variables declared outside the coforall can be read -by all tasks, while the variables declared inside, are available only to the -particular task. - -> ## Exercise 1 -> -> Would it be possible to print all the messages in the right order? Modify the -> code in the last example as required. -> -> Hint: you can use an array of strings declared in the main task, where all -> the concurrent tasks could write their messages in the corresponding -> position. Then, at the end, have the main task printing all elements of the -> array in order. -> -> > ## Solution -> > -> > The following code is a possible solution: -> > ~~~ -> > var x=1; -> > config var numoftasks=2; -> > var messages: [1..numoftasks] string; -> > -> > writeln("This is the main task: x = ",x); -> > -> > coforall taskid in 1..numoftasks do -> > { -> > var c=taskid+1; -> > var s="this is task "+taskid+": x + "+taskid+" = "+(x+taskid)+". My value of c is: "+c; -> > messages[taskid]=s; -> > } -> > -> > for i in 1..numoftasks do writeln(messages[i]); -> > writeln("this message won't appear until all tasks are done..."); -> > ~~~ -> > {: .source} -> > -> > ~~~ -> > chpl exercise_coforall.chpl -o exercise_coforall -> > ./exercise_coforall --numoftasks=5 -> > ~~~ -> > {: .bash} -> > -> > ~~~ -> > This is the main task: x = 1 -> > this is task 1: x + 1 = 2. My value of c is: 2 -> > this is task 2: x + 2 = 3. My value of c is: 3 -> > this is task 3: x + 3 = 4. My value of c is: 4 -> > this is task 4: x + 4 = 5. My value of c is: 5 -> > this is task 5: x + 5 = 6. My value of c is: 6 -> > this message won't appear until all tasks are done... -> > ~~~ -> > {: .output} -> > -> > Note that `+` is a **_polymorphic_** operand in Chapel. In this case it -> > concatenates `strings` with `integers` (which are transformed to strings). -> {: .solution} -{: .challenge} - -> ## Exercise 2 -> -> Consider the following code: -> -> ~~~ -> use Random; -> config const nelem=5000000000; -> var x: [1..n] int; -> fillRandom(x); //fill array with random numbers -> var mymax=0; -> -> // here put your code to find mymax -> -> writeln("the maximum value in x is: ",mymax); -> ~~~ -> {: .source} -> -> Write a parallel code to find the maximum value in the array x. -> -> > ## Solution -> > -> > ~~~ -> > config const numoftasks=12; -> > const n=nelem/numoftasks; -> > const r=nelem-n*numoftasks; -> > -> > var d: [0..numoftasks-1] real; -> > -> > coforall taskid in 0..numoftasks-1 do -> > { -> > var i: int; -> > var f: int; -> > if taskid > { -> > i=taskid*n+1+taskid; -> > f=taskid*n+n+taskid+1; -> > } -> > else -> > { -> > i=taskid*n+1+r; -> > f=taskid*n+n+r; -> > } -> > for c in i..f do -> > { -> > if x[c]>d[taskid] then d[taskid]=x[c]; -> > } -> > } -> > for i in 0..numoftasks-1 do -> > { -> > if d[i]>mymax then mymax=d[i]; -> > } -> > ~~~ -> > {: .source} -> > -> > ~~~ -> > > > chpl --fast exercise_coforall_2.chpl -o exercise_coforall_2 -> > > > ./exercise_coforall_2 -> > ~~~ -> > {: .bash} -> > -> > ~~~ -> > the maximum value in x is: 1.0 -> > ~~~ -> > {: .output} -> > -> > We use the coforall to spawn tasks that work concurrently in a fraction of -> > the array. The trick here is to determine, based on the _taskid_, the initial -> > and final indices that the task will use. Each task obtains the maximum in -> > its fraction of the array, and finally, after the coforall is done, the main -> > task obtains the maximum of the array from the maximums of all tasks. -> {: .solution} -{: .challenge} - -> ## Discussion -> -> Run the code of last Exercise using different number of tasks, and different -> sizes of the array _x_ to see how the execution time changes. For example: -> -> ~~~ -> time ./exercise_coforall_2 --nelem=3000 --numoftasks=4 -> ~~~ -> {: .bash} -> -> Discuss your observations. Is there a limit on how fast the code could run? -{: .discussion} - -> ## Try this... -> -> Substitute the code to find _mymax_ in the last exercise with: -> -> ~~~ -> mymax=max reduce x; -> ~~~ -> {: .source} -> -> Time the execution of the original code and this new one. How do they -> compare? -> -> > ## Key idea -> > -> > It is always a good idea to check whether there is _built-in_ functions or -> > methods in the used language, that can do what we want to do as efficiently -> > (or better) than our house-made code. In this case, the _reduce_ statement -> > reduces the given array to a single number using the given operation (in this -> > case max), and it is parallelized and optimised to have a very good -> > performance. -> {: .solution} -{: .challenge} - -The code in these last Exercises somehow _synchronise_ the tasks to obtain the -desired result. In addition, Chapel has specific mechanisms task -synchronisation, that could help us to achieve fine-grained parallelization. diff --git a/_extras/.gitkeep b/_extras/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/_extras/about.md b/_extras/about.md deleted file mode 100644 index aa7beea..0000000 --- a/_extras/about.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -layout: page -title: About -permalink: /about/ ---- -{% include carpentries.html %} diff --git a/_extras/discuss.md b/_extras/discuss.md deleted file mode 100644 index d62432a..0000000 --- a/_extras/discuss.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -layout: page -title: Discussion -permalink: /discuss/ ---- diff --git a/_extras/figures.md b/_extras/figures.md deleted file mode 100644 index de99a57..0000000 --- a/_extras/figures.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -layout: page -title: Figures -permalink: /figures/ ---- -{% include all_figures.html %} diff --git a/_extras/guide.md b/_extras/guide.md deleted file mode 100644 index 864ae16..0000000 --- a/_extras/guide.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -layout: page -title: "Instructor Notes" -permalink: /guide/ ---- - -This Chapel section is a full-day class. We expect it to be taught by a staff member from an HPC centre, and we assume that multi-locale Chapel has been already installed on the system that will be used for the hands-on exercises. For this reason, here we do not provide instructions for installing Chapel, as it can be done in several different ways, with details depending on the cluster's scheduler and physical interconnect, and it would be best to leave Chapel installation to knowledgeable HPC staff. - -Having said that, there are also ways to run Chapel entirely on attendees' laptops. This option is less than ideal, as there are some drawbacks to not having multi-locale Chapel run on actual physical compute nodes on a cluster. Here we mention two possible installation modes if an HPC cluster is not available: - -(1) You can install single-locale Chapel on a laptop and run it on multiple cores. In MacOS it is as simple as `brew install chapel`, and in Linux and Windows you might be able to find a package or compile Chapel from source. Obviously, you will not be able to do data decomposition and parallelism across multiple nodes. - -(2) You can run multi-locale Chapel inside a docker container (`docker pull chapel/chapel-gasnet`). This will imitate an HPC cluster (without the scheduler) on which you'll be able to do data parallelism. However, since you'll be running on the same set of physical laptop CPU cores, you will not get any acceleration from running multi-locale Chapel. This is probably the best option for learning all Chapel features without access to an actual cluster. diff --git a/_includes/all_figures.html b/_includes/all_figures.html deleted file mode 100644 index c341a40..0000000 --- a/_includes/all_figures.html +++ /dev/null @@ -1 +0,0 @@ - diff --git a/_includes/links.md b/_includes/links.md deleted file mode 100644 index 33cf70a..0000000 --- a/_includes/links.md +++ /dev/null @@ -1,46 +0,0 @@ -{% include base_path.html %} -[cc-by-human]: https://creativecommons.org/licenses/by/4.0/ -[cc-by-legal]: https://creativecommons.org/licenses/by/4.0/legalcode -[ci]: http://communityin.org/ -[coc-reporting]: https://docs.carpentries.org/topic_folders/policies/incident-reporting.html -[coc]: https://docs.carpentries.org/topic_folders/policies/code-of-conduct.html -[concept-maps]: https://carpentries.github.io/instructor-training/05-memory/ -[contrib-covenant]: https://contributor-covenant.org/ -[contributing]: {{ repo_url }}/blob/{{ source_branch }}/CONTRIBUTING.md -[cran-checkpoint]: https://cran.r-project.org/package=checkpoint -[cran-knitr]: https://cran.r-project.org/package=knitr -[cran-stringr]: https://cran.r-project.org/package=stringr -[dc-lessons]: http://www.datacarpentry.org/lessons/ -[email]: mailto:team@carpentries.org -[github-importer]: https://import.github.com/ -[importer]: https://github.com/new/import -[jekyll-collection]: https://jekyllrb.com/docs/collections/ -[jekyll-install]: https://jekyllrb.com/docs/installation/ -[jekyll-windows]: http://jekyll-windows.juthilo.com/ -[jekyll]: https://jekyllrb.com/ -[jupyter]: https://jupyter.org/ -[kramdown]: https://kramdown.gettalong.org/ -[lc-lessons]: https://librarycarpentry.org/lessons/ -[lesson-coc]: {{ relative_root_path }}{% link CODE_OF_CONDUCT.md %} -[lesson-example]: https://carpentries.github.io/lesson-example/ -[lesson-license]: {{ relative_root_path }}{% link LICENSE.md %} -[lesson-mainpage]: {{ relative_root_path }}{% link index.md %} -[mit-license]: https://opensource.org/licenses/mit-license.html -[morea]: https://morea-framework.github.io/ -[numfocus]: https://numfocus.org/ -[osi]: https://opensource.org -[pandoc]: https://pandoc.org/ -[paper-now]: https://github.com/PeerJ/paper-now -[python-gapminder]: https://swcarpentry.github.io/python-novice-gapminder/ -[pyyaml]: https://pypi.org/project/PyYAML/ -[r-markdown]: https://rmarkdown.rstudio.com/ -[rstudio]: https://www.rstudio.com/ -[ruby-install-guide]: https://www.ruby-lang.org/en/downloads/ -[ruby-installer]: https://rubyinstaller.org/ -[rubygems]: https://rubygems.org/pages/download/ -[styles]: https://github.com/carpentries/styles/ -[swc-lessons]: https://software-carpentry.org/lessons/ -[swc-releases]: https://github.com/swcarpentry/swc-releases -[training]: https://carpentries.github.io/instructor-training/ -[workshop-repo]: {{ site.workshop_repo }} -[yaml]: http://yaml.org/ diff --git a/aio.md b/aio.md deleted file mode 100644 index c5800f3..0000000 --- a/aio.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -permalink: /aio/index.html ---- - -{% include base_path.html %} - - -{% comment %} -Create anchor for each one of the episodes. -{% endcomment %} -{% for episode in site.episodes %} -
-{% endfor %} diff --git a/bin/chunk-options.R b/bin/chunk-options.R deleted file mode 100644 index 8e0d62a..0000000 --- a/bin/chunk-options.R +++ /dev/null @@ -1,70 +0,0 @@ -# These settings control the behavior of all chunks in the novice R materials. -# For example, to generate the lessons with all the output hidden, simply change -# `results` from "markup" to "hide". -# For more information on available chunk options, see -# http://yihui.name/knitr/options#chunk_options - -library("knitr") - -fix_fig_path <- function(pth) file.path("..", pth) - - -## We set the path for the figures globally below, so if we want to -## customize it for individual episodes, we can append a prefix to the -## global path. For instance, if we call knitr_fig_path("01-") in the -## first episode of the lesson, it will generate the figures in -## `fig/rmd-01-` -knitr_fig_path <- function(prefix) { - new_path <- paste0(opts_chunk$get("fig.path"), - prefix) - opts_chunk$set(fig.path = new_path) -} - -## We use the rmd- prefix for the figures generated by the lessons so -## they can be easily identified and deleted by `make clean-rmd`. The -## working directory when the lessons are generated is the root so the -## figures need to be saved in fig/, but when the site is generated, -## the episodes will be one level down. We fix the path using the -## `fig.process` option. - -opts_chunk$set(tidy = FALSE, results = "markup", comment = NA, - fig.align = "center", fig.path = "fig/rmd-", - fig.process = fix_fig_path, - fig.width = 8.5, fig.height = 8.5, - fig.retina = 2) - -# The hooks below add html tags to the code chunks and their output so that they -# are properly formatted when the site is built. - -hook_in <- function(x, options) { - lg <- tolower(options$engine) - style <- paste0(".language-", lg) - - stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: ", style, "}\n\n") -} - -hook_out <- function(x, options) { - x <- gsub("\n$", "", x) - stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: .output}\n\n") -} - -hook_error <- function(x, options) { - x <- gsub("\n$", "", x) - stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: .error}\n\n") -} - -hook_warning <- function(x, options) { - x <- gsub("\n$", "", x) - stringr::str_c("\n\n~~~\n", - paste0(x, collapse = "\n"), - "\n~~~\n{: .warning}\n\n") -} - -knit_hooks$set(source = hook_in, output = hook_out, warning = hook_warning, - error = hook_error, message = hook_out) diff --git a/bin/extract_figures.py b/bin/extract_figures.py deleted file mode 100755 index 63a7752..0000000 --- a/bin/extract_figures.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -import sys -import os -import glob -from optparse import OptionParser - -from util import Reporter, read_markdown, IMAGE_FILE_SUFFIX - -def main(): - """Main driver.""" - - args = parse_args() - images = [] - for filename in args.filenames: - images += get_images(args.parser, filename) - save(sys.stdout, images) - - -def parse_args(): - """Parse command-line arguments.""" - - parser = OptionParser() - parser.add_option('-p', '--parser', - default=None, - dest='parser', - help='path to Markdown parser') - - args, extras = parser.parse_args() - require(args.parser is not None, - 'Path to Markdown parser not provided') - require(extras, - 'No filenames specified') - - args.filenames = extras - return args - - -def get_filenames(source_dir): - """Get all filenames to be searched for images.""" - - return glob.glob(os.path.join(source_dir, '*.md')) - - -def get_images(parser, filename): - """Extract all images from file.""" - - content = read_markdown(parser, filename) - result = [] - find_image_nodes(content['doc'], result) - find_image_links(content['doc'], result) - return result - - -def find_image_nodes(doc, result): - """Find all nested nodes representing images.""" - - if (doc['type'] == 'img') or \ - ((doc['type'] == 'html_element') and (doc['value'] == 'img')): - alt = doc['attr'].get('alt', '') - result.append({'alt': alt, 'src': doc['attr']['src']}) - else: - for child in doc.get('children', []): - find_image_nodes(child, result) - - -def find_image_links(doc, result): - """Find all links to files in the 'fig' directory.""" - - if ((doc['type'] == 'a') and ('attr' in doc) and ('href' in doc['attr'])) \ - or \ - ((doc['type'] == 'html_element') and (doc['value'] == 'a') and ('href' in doc['attr'])): - path = doc['attr']['href'] - if os.path.splitext(path)[1].lower() in IMAGE_FILE_SUFFIX: - result.append({'alt':'', 'src': doc['attr']['href']}) - else: - for child in doc.get('children', []): - find_image_links(child, result) - - -def save(stream, images): - """Save results as Markdown.""" - - text = '\n
\n'.join(['

{0}

'.format(img['alt'], img['src']) for img in images]) - print(text, file=stream) - - -def require(condition, message): - """Fail if condition not met.""" - - if not condition: - print(message, file=sys.stderr) - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/bin/generate_md_episodes.R b/bin/generate_md_episodes.R deleted file mode 100644 index 0b4c69e..0000000 --- a/bin/generate_md_episodes.R +++ /dev/null @@ -1,71 +0,0 @@ -generate_md_episodes <- function() { - - library("methods") - - if (!require("remotes", quietly = TRUE)) { - install.packages("remotes", repos = c(CRAN = "https://cloud.r-project.org/")) - } - - if (!require("requirements", quietly = TRUE)) { - remotes::install_github("hadley/requirements") - } - - required_pkgs <- unique(c( - ## Packages for episodes - requirements:::req_dir("_episodes_rmd"), - ## Pacakges for tools - requirements:::req_dir("bin") - )) - - missing_pkgs <- setdiff(required_pkgs, rownames(installed.packages())) - - if (length(missing_pkgs)) { - message("Installing missing required packages: ", - paste(missing_pkgs, collapse=", ")) - install.packages(missing_pkgs) - } - - if (require("knitr") && packageVersion("knitr") < '1.9.19') - stop("knitr must be version 1.9.20 or higher") - - ## get the Rmd file to process from the command line, and generate the path for their respective outputs - args <- commandArgs(trailingOnly = TRUE) - if (!identical(length(args), 2L)) { - stop("input and output file must be passed to the script") - } - - src_rmd <- args[1] - dest_md <- args[2] - - ## knit the Rmd into markdown - knitr::knit(src_rmd, output = dest_md) - - # Read the generated md files and add comments advising not to edit them - vapply(dest_md, function(y) { - con <- file(y) - mdfile <- readLines(con) - if (mdfile[1] != "---") - stop("Input file does not have a valid header") - mdfile <- append(mdfile, "# Please do not edit this file directly; it is auto generated.", after = 1) - mdfile <- append(mdfile, paste("# Instead, please edit", - basename(y), "in _episodes_rmd/"), after = 2) - writeLines(mdfile, con) - close(con) - return(paste("Warning added to YAML header of", y)) - }, - character(1)) -} - -generate_md_episodes() -ead, please edit", basename(y), "in _episodes_rmd/"), - after = 2 - ) - writeLines(mdfile, con) - close(con) - return(paste("Warning added to YAML header of", y)) - } - - vapply(dest_md, add_no_edit_comment, character(1)) -} - -generate_md_episodes() diff --git a/bin/knit_lessons.sh b/bin/knit_lessons.sh deleted file mode 100755 index 194f124..0000000 --- a/bin/knit_lessons.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -# Only try running R to translate files if there are some files present. -# The Makefile passes in the names of files. - -if [ $# -ne 0 ] ; then - Rscript -e "source('bin/generate_md_episodes.R')" "$@" -fi diff --git a/bin/lesson_check.py b/bin/lesson_check.py deleted file mode 100755 index 5d0fa3d..0000000 --- a/bin/lesson_check.py +++ /dev/null @@ -1,564 +0,0 @@ -#!/usr/bin/env python3 - -""" -Check lesson files and their contents. -""" - - -import os -import glob -import re -from argparse import ArgumentParser - -from util import (Reporter, read_markdown, load_yaml, check_unwanted_files, - require) - -__version__ = '0.3' - -# Where to look for source Markdown files. -SOURCE_DIRS = ['', '_episodes', '_extras'] - -# Where to look for source Rmd files. -SOURCE_RMD_DIRS = ['_episodes_rmd'] - -# Required files: each entry is ('path': YAML_required). -# FIXME: We do not yet validate whether any files have the required -# YAML headers, but should in the future. -# The '%' is replaced with the source directory path for checking. -# Episodes are handled specially, and extra files in '_extras' are also handled -# specially. This list must include all the Markdown files listed in the -# 'bin/initialize' script. -REQUIRED_FILES = { - '%/CODE_OF_CONDUCT.md': True, - '%/CONTRIBUTING.md': False, - '%/LICENSE.md': True, - '%/README.md': False, - '%/_extras/discuss.md': True, - '%/_extras/guide.md': True, - '%/index.md': True, - '%/reference.md': True, - '%/setup.md': True, -} - -# Episode filename pattern. -P_EPISODE_FILENAME = re.compile(r'/_episodes/(\d\d)-[-\w]+.md$') - -# Pattern to match lines ending with whitespace. -P_TRAILING_WHITESPACE = re.compile(r'\s+$') - -# Pattern to match figure references in HTML. -P_FIGURE_REFS = re.compile(r']+src="([^"]+)"[^>]*>') - -# Pattern to match internally-defined Markdown links. -P_INTERNAL_LINK_REF = re.compile(r'\[([^\]]+)\]\[([^\]]+)\]') - -# Pattern to match reference links (to resolve internally-defined references). -P_INTERNAL_LINK_DEF = re.compile(r'^\[([^\]]+)\]:\s*(.+)') - -# Pattern to match {% include ... %} statements -P_INTERNAL_INCLUDE_LINK = re.compile(r'^{% include ([^ ]*) %}$') - -# What kinds of blockquotes are allowed? -KNOWN_BLOCKQUOTES = { - 'callout', - 'caution', - 'challenge', - 'checklist', - 'discussion', - 'keypoints', - 'objectives', - 'prereq', - 'quotation', - 'solution', - 'testimonial', - 'warning' -} - -# What kinds of code fragments are allowed? -KNOWN_CODEBLOCKS = { - 'error', - 'output', - 'source', - 'language-bash', - 'html', - 'language-make', - 'language-matlab', - 'language-python', - 'language-r', - 'language-shell', - 'language-sql', - 'bash', - 'python' -} - -# What fields are required in teaching episode metadata? -TEACHING_METADATA_FIELDS = { - ('title', str), - ('teaching', int), - ('exercises', int), - ('questions', list), - ('objectives', list), - ('keypoints', list) -} - -# What fields are required in break episode metadata? -BREAK_METADATA_FIELDS = { - ('layout', str), - ('title', str), - ('break', int) -} - -# How long are lines allowed to be? -# Please keep this in sync with .editorconfig! -MAX_LINE_LEN = 100 - - -def main(): - """Main driver.""" - - args = parse_args() - args.reporter = Reporter() - check_config(args.reporter, args.source_dir) - check_source_rmd(args.reporter, args.source_dir, args.parser) - args.references = read_references(args.reporter, args.reference_path) - - docs = read_all_markdown(args.source_dir, args.parser) - check_fileset(args.source_dir, args.reporter, list(docs.keys())) - check_unwanted_files(args.source_dir, args.reporter) - for filename in list(docs.keys()): - checker = create_checker(args, filename, docs[filename]) - checker.check() - - args.reporter.report() - if args.reporter.messages and not args.permissive: - exit(1) - - -def parse_args(): - """Parse command-line arguments.""" - - parser = ArgumentParser(description="""Check episode files in a lesson.""") - parser.add_argument('-l', '--linelen', - default=False, - action="store_true", - dest='line_lengths', - help='Check line lengths') - parser.add_argument('-p', '--parser', - default=None, - dest='parser', - help='path to Markdown parser') - parser.add_argument('-r', '--references', - default=None, - dest='reference_path', - help='path to Markdown file of external references') - parser.add_argument('-s', '--source', - default=os.curdir, - dest='source_dir', - help='source directory') - parser.add_argument('-w', '--whitespace', - default=False, - action="store_true", - dest='trailing_whitespace', - help='Check for trailing whitespace') - parser.add_argument('--permissive', - default=False, - action="store_true", - dest='permissive', - help='Do not raise an error even if issues are detected') - - args, extras = parser.parse_known_args() - require(args.parser is not None, - 'Path to Markdown parser not provided') - require(not extras, - 'Unexpected trailing command-line arguments "{0}"'.format(extras)) - - return args - - -def check_config(reporter, source_dir): - """Check configuration file.""" - - config_file = os.path.join(source_dir, '_config.yml') - config = load_yaml(config_file) - reporter.check_field(config_file, 'configuration', - config, 'kind', 'lesson') - reporter.check_field(config_file, 'configuration', - config, 'carpentry', ('swc', 'dc', 'lc', 'cp', 'incubator')) - reporter.check_field(config_file, 'configuration', config, 'title') - reporter.check_field(config_file, 'configuration', config, 'email') - - for defaults in [ - {'values': {'root': '.', 'layout': 'page'}}, - {'values': {'root': '..', 'layout': 'episode'}, 'scope': {'type': 'episodes', 'path': ''}}, - {'values': {'root': '..', 'layout': 'page'}, 'scope': {'type': 'extras', 'path': ''}} - ]: - reporter.check(defaults in config.get('defaults', []), - 'configuration', - '"root" not set to "." in configuration') - -def check_source_rmd(reporter, source_dir, parser): - """Check that Rmd episode files include `source: Rmd`""" - - episode_rmd_dir = [os.path.join(source_dir, d) for d in SOURCE_RMD_DIRS] - episode_rmd_files = [os.path.join(d, '*.Rmd') for d in episode_rmd_dir] - results = {} - for pat in episode_rmd_files: - for f in glob.glob(pat): - data = read_markdown(parser, f) - dy = data['metadata'] - if dy: - reporter.check_field(f, 'episode_rmd', - dy, 'source', 'Rmd') - -def read_references(reporter, ref_path): - """Read shared file of reference links, returning dictionary of valid references - {symbolic_name : URL} - """ - - if not ref_path: - raise Warning("No filename has been provided.") - - result = {} - urls_seen = set() - - with open(ref_path, 'r') as reader: - for (num, line) in enumerate(reader, 1): - - if P_INTERNAL_INCLUDE_LINK.search(line): continue - - m = P_INTERNAL_LINK_DEF.search(line) - - message = '{}: {} not a valid reference: {}' - require(m, message.format(ref_path, num, line.rstrip())) - - name = m.group(1) - url = m.group(2) - - message = 'Empty reference at {0}:{1}' - require(name, message.format(ref_path, num)) - - unique_name = name not in result - unique_url = url not in urls_seen - - reporter.check(unique_name, - ref_path, - 'Duplicate reference name {0} at line {1}', - name, num) - - reporter.check(unique_url, - ref_path, - 'Duplicate definition of URL {0} at line {1}', - url, num) - - result[name] = url - urls_seen.add(url) - - return result - - -def read_all_markdown(source_dir, parser): - """Read source files, returning - {path : {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}} - """ - - all_dirs = [os.path.join(source_dir, d) for d in SOURCE_DIRS] - all_patterns = [os.path.join(d, '*.md') for d in all_dirs] - result = {} - for pat in all_patterns: - for filename in glob.glob(pat): - data = read_markdown(parser, filename) - if data: - result[filename] = data - return result - - -def check_fileset(source_dir, reporter, filenames_present): - """Are all required files present? Are extraneous files present?""" - - # Check files with predictable names. - required = [p.replace('%', source_dir) for p in REQUIRED_FILES] - missing = set(required) - set(filenames_present) - for m in missing: - reporter.add(None, 'Missing required file {0}', m) - - # Check episode files' names. - seen = [] - for filename in filenames_present: - if '_episodes' not in filename: - continue - m = P_EPISODE_FILENAME.search(filename) - if m and m.group(1): - seen.append(m.group(1)) - else: - reporter.add( - None, 'Episode {0} has badly-formatted filename', filename) - - # Check for duplicate episode numbers. - reporter.check(len(seen) == len(set(seen)), - None, - 'Duplicate episode numbers {0} vs {1}', - sorted(seen), sorted(set(seen))) - - # Check that numbers are consecutive. - seen = sorted([int(s) for s in seen]) - clean = True - for i in range(len(seen) - 1): - clean = clean and ((seen[i+1] - seen[i]) == 1) - reporter.check(clean, - None, - 'Missing or non-consecutive episode numbers {0}', - seen) - - -def create_checker(args, filename, info): - """Create appropriate checker for file.""" - - for (pat, cls) in CHECKERS: - if pat.search(filename): - return cls(args, filename, **info) - return NotImplemented - -class CheckBase: - """Base class for checking Markdown files.""" - - def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): - """Cache arguments for checking.""" - - self.args = args - self.reporter = self.args.reporter # for convenience - self.filename = filename - self.metadata = metadata - self.metadata_len = metadata_len - self.text = text - self.lines = lines - self.doc = doc - - self.layout = None - - def check(self): - """Run tests.""" - - self.check_metadata() - self.check_line_lengths() - self.check_trailing_whitespace() - self.check_blockquote_classes() - self.check_codeblock_classes() - self.check_defined_link_references() - - def check_metadata(self): - """Check the YAML metadata.""" - - self.reporter.check(self.metadata is not None, - self.filename, - 'Missing metadata entirely') - - if self.metadata and (self.layout is not None): - self.reporter.check_field( - self.filename, 'metadata', self.metadata, 'layout', self.layout) - - def check_line_lengths(self): - """Check the raw text of the lesson body.""" - - if self.args.line_lengths: - over = [i for (i, l, n) in self.lines if ( - n > MAX_LINE_LEN) and (not l.startswith('!'))] - self.reporter.check(not over, - self.filename, - 'Line(s) too long: {0}', - ', '.join([str(i) for i in over])) - - def check_trailing_whitespace(self): - """Check for whitespace at the ends of lines.""" - - if self.args.trailing_whitespace: - trailing = [ - i for (i, l, n) in self.lines if P_TRAILING_WHITESPACE.match(l)] - self.reporter.check(not trailing, - self.filename, - 'Line(s) end with whitespace: {0}', - ', '.join([str(i) for i in trailing])) - - def check_blockquote_classes(self): - """Check that all blockquotes have known classes.""" - - for node in self.find_all(self.doc, {'type': 'blockquote'}): - cls = self.get_val(node, 'attr', 'class') - self.reporter.check(cls in KNOWN_BLOCKQUOTES, - (self.filename, self.get_loc(node)), - 'Unknown or missing blockquote type {0}', - cls) - - def check_codeblock_classes(self): - """Check that all code blocks have known classes.""" - - for node in self.find_all(self.doc, {'type': 'codeblock'}): - cls = self.get_val(node, 'attr', 'class') - self.reporter.check(cls in KNOWN_CODEBLOCKS, - (self.filename, self.get_loc(node)), - 'Unknown or missing code block type {0}', - cls) - - def check_defined_link_references(self): - """Check that defined links resolve in the file. - - Internally-defined links match the pattern [text][label]. - """ - - result = set() - for node in self.find_all(self.doc, {'type': 'text'}): - for match in P_INTERNAL_LINK_REF.findall(node['value']): - text = match[0] - link = match[1] - if link not in self.args.references: - result.add('"{0}"=>"{1}"'.format(text, link)) - self.reporter.check(not result, - self.filename, - 'Internally-defined links may be missing definitions: {0}', - ', '.join(sorted(result))) - - def find_all(self, node, pattern, accum=None): - """Find all matches for a pattern.""" - - assert isinstance(pattern, dict), 'Patterns must be dictionaries' - if accum is None: - accum = [] - if self.match(node, pattern): - accum.append(node) - for child in node.get('children', []): - self.find_all(child, pattern, accum) - return accum - - def match(self, node, pattern): - """Does this node match the given pattern?""" - - for key in pattern: - if key not in node: - return False - val = pattern[key] - if isinstance(val, str): - if node[key] != val: - return False - elif isinstance(val, dict): - if not self.match(node[key], val): - return False - return True - - @staticmethod - def get_val(node, *chain): - """Get value one or more levels down.""" - - curr = node - for selector in chain: - curr = curr.get(selector, None) - if curr is None: - break - return curr - - def get_loc(self, node): - """Convenience method to get node's line number.""" - - result = self.get_val(node, 'options', 'location') - if self.metadata_len is not None: - result += self.metadata_len - return result - - -class CheckNonJekyll(CheckBase): - """Check a file that isn't translated by Jekyll.""" - - def check_metadata(self): - self.reporter.check(self.metadata is None, - self.filename, - 'Unexpected metadata') - - -class CheckIndex(CheckBase): - """Check the main index page.""" - - def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): - super().__init__(args, filename, metadata, metadata_len, text, lines, doc) - self.layout = 'lesson' - - def check_metadata(self): - super().check_metadata() - self.reporter.check(self.metadata.get('root', '') == '.', - self.filename, - 'Root not set to "."') - - -class CheckEpisode(CheckBase): - """Check an episode page.""" - - def check(self): - """Run extra tests.""" - - super().check() - self.check_reference_inclusion() - - def check_metadata(self): - super().check_metadata() - if self.metadata: - if 'layout' in self.metadata: - if self.metadata['layout'] == 'break': - self.check_metadata_fields(BREAK_METADATA_FIELDS) - else: - self.reporter.add(self.filename, - 'Unknown episode layout "{0}"', - self.metadata['layout']) - else: - self.check_metadata_fields(TEACHING_METADATA_FIELDS) - - def check_metadata_fields(self, expected): - """Check metadata fields.""" - for (name, type_) in expected: - if name not in self.metadata: - self.reporter.add(self.filename, - 'Missing metadata field {0}', - name) - elif not isinstance(self.metadata[name], type_): - self.reporter.add(self.filename, - '"{0}" has wrong type in metadata ({1} instead of {2})', - name, type(self.metadata[name]), type_) - - def check_reference_inclusion(self): - """Check that links file has been included.""" - - if not self.args.reference_path: - return - - for (i, last_line, line_len) in reversed(self.lines): - if last_line: - break - - require(last_line, - 'No non-empty lines in {0}'.format(self.filename)) - - - -class CheckReference(CheckBase): - """Check the reference page.""" - - def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): - super().__init__(args, filename, metadata, metadata_len, text, lines, doc) - self.layout = 'reference' - - -class CheckGeneric(CheckBase): - """Check a generic page.""" - - def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): - super().__init__(args, filename, metadata, metadata_len, text, lines, doc) - - -CHECKERS = [ - (re.compile(r'CONTRIBUTING\.md'), CheckNonJekyll), - (re.compile(r'README\.md'), CheckNonJekyll), - (re.compile(r'index\.md'), CheckIndex), - (re.compile(r'reference\.md'), CheckReference), - (re.compile(r'_episodes/.*\.md'), CheckEpisode), - (re.compile(r'.*\.md'), CheckGeneric) -] - - -if __name__ == '__main__': - main() diff --git a/bin/lesson_initialize.py b/bin/lesson_initialize.py deleted file mode 100755 index 65079c1..0000000 --- a/bin/lesson_initialize.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 - -"""Initialize a newly-created repository.""" - - -import sys -import os -import shutil - -BOILERPLATE = ( - '.travis.yml', - 'AUTHORS', - 'CITATION', - 'CONTRIBUTING.md', - 'MAINTENANCE.md', - 'README.md', - '_config.yml', - '_episodes/01-introduction.md', - '_extras/about.md', - '_extras/discuss.md', - '_extras/figures.md', - '_extras/guide.md', - 'aio.md', - 'index.md', - 'reference.md', - 'setup.md', -) - - -def main(): - """Check for collisions, then create.""" - - # Check. - errors = False - for path in BOILERPLATE: - if os.path.exists(path): - print('Warning: {0} already exists.'.format(path), file=sys.stderr) - errors = True - if errors: - print('**Exiting without creating files.**', file=sys.stderr) - sys.exit(1) - - # Create. - for path in BOILERPLATE: - shutil.copyfile( - "bin/boilerplate/{}".format(path), - path - ) - - -if __name__ == '__main__': - main() diff --git a/bin/markdown_ast.rb b/bin/markdown_ast.rb deleted file mode 100755 index 2ef3f77..0000000 --- a/bin/markdown_ast.rb +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# Use Kramdown parser to produce AST for Markdown document. - -require 'kramdown' -require 'kramdown-parser-gfm' -require 'json' - -markdown = $stdin.read -doc = Kramdown::Document.new(markdown, input: 'GFM', hard_wrap: false) -tree = doc.to_hash_a_s_t -puts JSON.pretty_generate(tree) diff --git a/bin/repo_check.py b/bin/repo_check.py deleted file mode 100755 index 9bf5c59..0000000 --- a/bin/repo_check.py +++ /dev/null @@ -1,180 +0,0 @@ -""" -Check repository settings. -""" - - -import sys -import os -from subprocess import Popen, PIPE -import re -from argparse import ArgumentParser - -from util import Reporter, require - -# Import this way to produce a more useful error message. -try: - import requests -except ImportError: - print('Unable to import requests module: please install requests', file=sys.stderr) - sys.exit(1) - - -# Pattern to match Git command-line output for remotes => (user name, project name). -P_GIT_REMOTE = re.compile(r'upstream\s+(?:https://|git@)github.com[:/]([^/]+)/([^.]+)(\.git)?\s+\(fetch\)') - -# Repository URL format string. -F_REPO_URL = 'https://github.com/{0}/{1}/' - -# Pattern to match repository URLs => (user name, project name) -P_REPO_URL = re.compile(r'https?://github\.com/([^.]+)/([^/]+)/?') - -# API URL format string. -F_API_URL = 'https://api.github.com/repos/{0}/{1}/labels' - -# Expected labels and colors. -EXPECTED = { - 'help wanted': 'dcecc7', - 'status:in progress': '9bcc65', - 'status:changes requested': '679f38', - 'status:wait': 'fff2df', - 'status:refer to cac': 'ffdfb2', - 'status:need more info': 'ee6c00', - 'status:blocked': 'e55100', - 'status:out of scope': 'eeeeee', - 'status:duplicate': 'bdbdbd', - 'type:typo text': 'f8bad0', - 'type:bug': 'eb3f79', - 'type:formatting': 'ac1357', - 'type:template and tools': '7985cb', - 'type:instructor guide': '00887a', - 'type:discussion': 'b2e5fc', - 'type:enhancement': '7fdeea', - 'type:clarification': '00acc0', - 'type:teaching example': 'ced8dc', - 'good first issue': 'ffeb3a', - 'high priority': 'd22e2e' -} - - -def main(): - """ - Main driver. - """ - - args = parse_args() - reporter = Reporter() - repo_url = get_repo_url(args.repo_url) - check_labels(reporter, repo_url) - reporter.report() - - -def parse_args(): - """ - Parse command-line arguments. - """ - - parser = ArgumentParser(description="""Check repository settings.""") - parser.add_argument('-r', '--repo', - default=None, - dest='repo_url', - help='repository URL') - parser.add_argument('-s', '--source', - default=os.curdir, - dest='source_dir', - help='source directory') - - args, extras = parser.parse_known_args() - require(not extras, - 'Unexpected trailing command-line arguments "{0}"'.format(extras)) - - return args - - -def get_repo_url(repo_url): - """ - Figure out which repository to query. - """ - - # Explicitly specified. - if repo_url is not None: - return repo_url - - # Guess. - cmd = 'git remote -v' - p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, - close_fds=True, universal_newlines=True, encoding='utf-8') - stdout_data, stderr_data = p.communicate() - stdout_data = stdout_data.split('\n') - matches = [P_GIT_REMOTE.match(line) for line in stdout_data] - matches = [m for m in matches if m is not None] - require(len(matches) == 1, - 'Unexpected output from git remote command: "{0}"'.format(matches)) - - username = matches[0].group(1) - require( - username, 'empty username in git remote output {0}'.format(matches[0])) - - project_name = matches[0].group(2) - require( - username, 'empty project name in git remote output {0}'.format(matches[0])) - - url = F_REPO_URL.format(username, project_name) - return url - - -def check_labels(reporter, repo_url): - """ - Check labels in repository. - """ - - actual = get_labels(repo_url) - extra = set(actual.keys()) - set(EXPECTED.keys()) - - reporter.check(not extra, - None, - 'Extra label(s) in repository {0}: {1}', - repo_url, ', '.join(sorted(extra))) - - missing = set(EXPECTED.keys()) - set(actual.keys()) - reporter.check(not missing, - None, - 'Missing label(s) in repository {0}: {1}', - repo_url, ', '.join(sorted(missing))) - - overlap = set(EXPECTED.keys()).intersection(set(actual.keys())) - for name in sorted(overlap): - reporter.check(EXPECTED[name].lower() == actual[name].lower(), - None, - 'Color mis-match for label {0} in {1}: expected {2}, found {3}', - name, repo_url, EXPECTED[name], actual[name]) - - -def get_labels(repo_url): - """ - Get actual labels from repository. - """ - - m = P_REPO_URL.match(repo_url) - require( - m, 'repository URL {0} does not match expected pattern'.format(repo_url)) - - username = m.group(1) - require(username, 'empty username in repository URL {0}'.format(repo_url)) - - project_name = m.group(2) - require( - username, 'empty project name in repository URL {0}'.format(repo_url)) - - url = F_API_URL.format(username, project_name) - r = requests.get(url) - require(r.status_code == 200, - 'Request for {0} failed with {1}'.format(url, r.status_code)) - - result = {} - for entry in r.json(): - result[entry['name']] = entry['color'] - return result - - -if __name__ == '__main__': - main() diff --git a/bin/run-make-docker-serve.sh b/bin/run-make-docker-serve.sh deleted file mode 100644 index 1e09178..0000000 --- a/bin/run-make-docker-serve.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail -set -o nounset - - -bundle install -bundle update -exec bundle exec jekyll serve --host 0.0.0.0 diff --git a/bin/test_lesson_check.py b/bin/test_lesson_check.py deleted file mode 100755 index 960059e..0000000 --- a/bin/test_lesson_check.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 - -import unittest - -import lesson_check -import util - - -class TestFileList(unittest.TestCase): - def setUp(self): - self.reporter = util.Reporter() # TODO: refactor reporter class. - - def test_file_list_has_expected_entries(self): - # For first pass, simply assume that all required files are present - all_filenames = [filename.replace('%', '') - for filename in lesson_check.REQUIRED_FILES] - - lesson_check.check_fileset('', self.reporter, all_filenames) - self.assertEqual(len(self.reporter.messages), 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/bin/util.py b/bin/util.py deleted file mode 100644 index 2705f77..0000000 --- a/bin/util.py +++ /dev/null @@ -1,188 +0,0 @@ -import sys -import os -import json -from subprocess import Popen, PIPE - -# Import this way to produce a more useful error message. -try: - import yaml -except ImportError: - print('Unable to import YAML module: please install PyYAML', file=sys.stderr) - sys.exit(1) - - -# Things an image file's name can end with. -IMAGE_FILE_SUFFIX = { - '.gif', - '.jpg', - '.png', - '.svg' -} - -# Files that shouldn't be present. -UNWANTED_FILES = [ - '.nojekyll' -] - -# Marker to show that an expected value hasn't been provided. -# (Can't use 'None' because that might be a legitimate value.) -REPORTER_NOT_SET = [] - - -class Reporter: - """Collect and report errors.""" - - def __init__(self): - """Constructor.""" - self.messages = [] - - def check_field(self, filename, name, values, key, expected=REPORTER_NOT_SET): - """Check that a dictionary has an expected value.""" - - if key not in values: - self.add(filename, '{0} does not contain {1}', name, key) - elif expected is REPORTER_NOT_SET: - pass - elif type(expected) in (tuple, set, list): - if values[key] not in expected: - self.add( - filename, '{0} {1} value {2} is not in {3}', name, key, values[key], expected) - elif values[key] != expected: - self.add(filename, '{0} {1} is {2} not {3}', - name, key, values[key], expected) - - def check(self, condition, location, fmt, *args): - """Append error if condition not met.""" - - if not condition: - self.add(location, fmt, *args) - - def add(self, location, fmt, *args): - """Append error unilaterally.""" - - self.messages.append((location, fmt.format(*args))) - - @staticmethod - def pretty(item): - location, message = item - if isinstance(location, type(None)): - return message - elif isinstance(location, str): - return location + ': ' + message - elif isinstance(location, tuple): - return '{0}:{1}: '.format(*location) + message - - print('Unknown item "{0}"'.format(item), file=sys.stderr) - return NotImplemented - - @staticmethod - def key(item): - location, message = item - if isinstance(location, type(None)): - return ('', -1, message) - elif isinstance(location, str): - return (location, -1, message) - elif isinstance(location, tuple): - return (location[0], location[1], message) - - print('Unknown item "{0}"'.format(item), file=sys.stderr) - return NotImplemented - - def report(self, stream=sys.stdout): - """Report all messages in order.""" - - if not self.messages: - return - - for m in sorted(self.messages, key=self.key): - print(self.pretty(m), file=stream) - - -def read_markdown(parser, path): - """ - Get YAML and AST for Markdown file, returning - {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}. - """ - - # Split and extract YAML (if present). - with open(path, 'r') as reader: - body = reader.read() - metadata_raw, metadata_yaml, body = split_metadata(path, body) - - # Split into lines. - metadata_len = 0 if metadata_raw is None else metadata_raw.count('\n') - lines = [(metadata_len+i+1, line, len(line)) - for (i, line) in enumerate(body.split('\n'))] - - # Parse Markdown. - cmd = 'bundle exec ruby {0}'.format(parser) - p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, - close_fds=True, universal_newlines=True) - stdout_data, stderr_data = p.communicate(body) - doc = json.loads(stdout_data) - - return { - 'metadata': metadata_yaml, - 'metadata_len': metadata_len, - 'text': body, - 'lines': lines, - 'doc': doc - } - - -def split_metadata(path, text): - """ - Get raw (text) metadata, metadata as YAML, and rest of body. - If no metadata, return (None, None, body). - """ - - metadata_raw = None - metadata_yaml = None - - pieces = text.split('---', 2) - if len(pieces) == 3: - metadata_raw = pieces[1] - text = pieces[2] - try: - metadata_yaml = yaml.load(metadata_raw, Loader=yaml.SafeLoader) - except yaml.YAMLError as e: - print('Unable to parse YAML header in {0}:\n{1}'.format( - path, e), file=sys.stderr) - sys.exit(1) - - return metadata_raw, metadata_yaml, text - - -def load_yaml(filename): - """ - Wrapper around YAML loading so that 'import yaml' is only needed - in one file. - """ - - try: - with open(filename, 'r') as reader: - return yaml.load(reader, Loader=yaml.SafeLoader) - except (yaml.YAMLError, IOError) as e: - print('Unable to load YAML file {0}:\n{1}'.format( - filename, e), file=sys.stderr) - sys.exit(1) - - -def check_unwanted_files(dir_path, reporter): - """ - Check that unwanted files are not present. - """ - - for filename in UNWANTED_FILES: - path = os.path.join(dir_path, filename) - reporter.check(not os.path.exists(path), - path, - "Unwanted file found") - - -def require(condition, message): - """Fail if condition not met.""" - - if not condition: - print(message, file=sys.stderr) - sys.exit(1) diff --git a/bin/workshop_check.py b/bin/workshop_check.py deleted file mode 100755 index 15d954a..0000000 --- a/bin/workshop_check.py +++ /dev/null @@ -1,418 +0,0 @@ -'''Check that a workshop's index.html metadata is valid. See the -docstrings on the checking functions for a summary of the checks. -''' - - -import sys -import os -import re -from datetime import date -from util import Reporter, split_metadata, load_yaml, check_unwanted_files - -# Metadata field patterns. -EMAIL_PATTERN = r'[^@]+@[^@]+\.[^@]+' -HUMANTIME_PATTERN = r'((0?[1-9]|1[0-2]):[0-5]\d(am|pm)(-|to)(0?[1-9]|1[0-2]):[0-5]\d(am|pm))|((0?\d|1\d|2[0-3]):[0-5]\d(-|to)(0?\d|1\d|2[0-3]):[0-5]\d)' -EVENTBRITE_PATTERN = r'\d{9,10}' -URL_PATTERN = r'https?://.+' - -# Defaults. -CARPENTRIES = ("dc", "swc", "lc", "cp") -DEFAULT_CONTACT_EMAIL = 'admin@software-carpentry.org' - -USAGE = 'Usage: "workshop_check.py path/to/root/directory"' - -# Country and language codes. Note that codes mean different things: 'ar' -# is 'Arabic' as a language but 'Argentina' as a country. - -ISO_COUNTRY = [ - 'ad', 'ae', 'af', 'ag', 'ai', 'al', 'am', 'an', 'ao', 'aq', 'ar', 'as', - 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd', 'be', 'bf', 'bg', 'bh', - 'bi', 'bj', 'bm', 'bn', 'bo', 'br', 'bs', 'bt', 'bv', 'bw', 'by', 'bz', - 'ca', 'cc', 'cd', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'cm', 'cn', 'co', - 'cr', 'cu', 'cv', 'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm', 'do', 'dz', - 'ec', 'ee', 'eg', 'eh', 'er', 'es', 'et', 'eu', 'fi', 'fj', 'fk', 'fm', - 'fo', 'fr', 'ga', 'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'gm', - 'gn', 'gp', 'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk', 'hm', 'hn', - 'hr', 'ht', 'hu', 'id', 'ie', 'il', 'im', 'in', 'io', 'iq', 'ir', 'is', - 'it', 'je', 'jm', 'jo', 'jp', 'ke', 'kg', 'kh', 'ki', 'km', 'kn', 'kp', - 'kr', 'kw', 'ky', 'kz', 'la', 'lb', 'lc', 'li', 'lk', 'lr', 'ls', 'lt', - 'lu', 'lv', 'ly', 'ma', 'mc', 'md', 'me', 'mg', 'mh', 'mk', 'ml', 'mm', - 'mn', 'mo', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu', 'mv', 'mw', 'mx', 'my', - 'mz', 'na', 'nc', 'ne', 'nf', 'ng', 'ni', 'nl', 'no', 'np', 'nr', 'nu', - 'nz', 'om', 'pa', 'pe', 'pf', 'pg', 'ph', 'pk', 'pl', 'pm', 'pn', 'pr', - 'ps', 'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru', 'rw', 'sa', 'sb', - 'sc', 'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl', 'sm', 'sn', 'so', - 'sr', 'st', 'sv', 'sy', 'sz', 'tc', 'td', 'tf', 'tg', 'th', 'tj', 'tk', - 'tl', 'tm', 'tn', 'to', 'tr', 'tt', 'tv', 'tw', 'tz', 'ua', 'ug', 'um', - 'us', 'uy', 'uz', 'va', 'vc', 've', 'vg', 'vi', 'vn', 'vu', 'wf', 'ws', - 'ye', 'yt', 'za', 'zm', 'zw' -] - -ISO_LANGUAGE = [ - 'aa', 'ab', 'ae', 'af', 'ak', 'am', 'an', 'ar', 'as', 'av', 'ay', 'az', - 'ba', 'be', 'bg', 'bh', 'bi', 'bm', 'bn', 'bo', 'br', 'bs', 'ca', 'ce', - 'ch', 'co', 'cr', 'cs', 'cu', 'cv', 'cy', 'da', 'de', 'dv', 'dz', 'ee', - 'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'ff', 'fi', 'fj', 'fo', 'fr', - 'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'gv', 'ha', 'he', 'hi', 'ho', 'hr', - 'ht', 'hu', 'hy', 'hz', 'ia', 'id', 'ie', 'ig', 'ii', 'ik', 'io', 'is', - 'it', 'iu', 'ja', 'jv', 'ka', 'kg', 'ki', 'kj', 'kk', 'kl', 'km', 'kn', - 'ko', 'kr', 'ks', 'ku', 'kv', 'kw', 'ky', 'la', 'lb', 'lg', 'li', 'ln', - 'lo', 'lt', 'lu', 'lv', 'mg', 'mh', 'mi', 'mk', 'ml', 'mn', 'mr', 'ms', - 'mt', 'my', 'na', 'nb', 'nd', 'ne', 'ng', 'nl', 'nn', 'no', 'nr', 'nv', - 'ny', 'oc', 'oj', 'om', 'or', 'os', 'pa', 'pi', 'pl', 'ps', 'pt', 'qu', - 'rm', 'rn', 'ro', 'ru', 'rw', 'sa', 'sc', 'sd', 'se', 'sg', 'si', 'sk', - 'sl', 'sm', 'sn', 'so', 'sq', 'sr', 'ss', 'st', 'su', 'sv', 'sw', 'ta', - 'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tn', 'to', 'tr', 'ts', 'tt', 'tw', - 'ty', 'ug', 'uk', 'ur', 'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi', - 'yo', 'za', 'zh', 'zu' -] - - -def look_for_fixme(func): - """Decorator to fail test if text argument starts with "FIXME".""" - - def inner(arg): - if (arg is not None) and \ - isinstance(arg, str) and \ - arg.lstrip().startswith('FIXME'): - return False - return func(arg) - return inner - - -@look_for_fixme -def check_layout(layout): - '''"layout" in YAML header must be "workshop".''' - - return layout == 'workshop' - - -@look_for_fixme -def check_carpentry(layout): - '''"carpentry" in YAML header must be "dc", "swc", "lc", or "cp".''' - - return layout in CARPENTRIES - - -@look_for_fixme -def check_country(country): - '''"country" must be a lowercase ISO-3166 two-letter code.''' - - return country in ISO_COUNTRY - - -@look_for_fixme -def check_language(language): - '''"language" must be a lowercase ISO-639 two-letter code.''' - - return language in ISO_LANGUAGE - - -@look_for_fixme -def check_humandate(date): - """ - 'humandate' must be a human-readable date with a 3-letter month - and 4-digit year. Examples include 'Feb 18-20, 2025' and 'Feb 18 - and 20, 2025'. It may be in languages other than English, but the - month name should be kept short to aid formatting of the main - Carpentries web site. - """ - - if ',' not in date: - return False - - month_dates, year = date.split(',') - - # The first three characters of month_dates are not empty - month = month_dates[:3] - if any(char == ' ' for char in month): - return False - - # But the fourth character is empty ("February" is illegal) - if month_dates[3] != ' ': - return False - - # year contains *only* numbers - try: - int(year) - except: - return False - - return True - - -@look_for_fixme -def check_humantime(time): - """ - 'humantime' is a human-readable start and end time for the - workshop, such as '09:00 - 16:00'. - """ - - return bool(re.match(HUMANTIME_PATTERN, time.replace(' ', ''))) - - -def check_date(this_date): - """ - 'startdate' and 'enddate' are machine-readable start and end dates - for the workshop, and must be in YYYY-MM-DD format, e.g., - '2015-07-01'. - """ - - # YAML automatically loads valid dates as datetime.date. - return isinstance(this_date, date) - - -@look_for_fixme -def check_latitude_longitude(latlng): - """ - 'latlng' must be a valid latitude and longitude represented as two - floating-point numbers separated by a comma. - """ - - try: - lat, lng = latlng.split(',') - lat = float(lat) - lng = float(lng) - return (-90.0 <= lat <= 90.0) and (-180.0 <= lng <= 180.0) - except ValueError: - return False - - -def check_instructors(instructors): - """ - 'instructor' must be a non-empty comma-separated list of quoted - names, e.g. ['First name', 'Second name', ...']. Do not use 'TBD' - or other placeholders. - """ - - # YAML automatically loads list-like strings as lists. - return isinstance(instructors, list) and len(instructors) > 0 - - -def check_helpers(helpers): - """ - 'helper' must be a comma-separated list of quoted names, - e.g. ['First name', 'Second name', ...']. The list may be empty. - Do not use 'TBD' or other placeholders. - """ - - # YAML automatically loads list-like strings as lists. - return isinstance(helpers, list) and len(helpers) >= 0 - - -@look_for_fixme -def check_emails(emails): - """ - 'emails' must be a comma-separated list of valid email addresses. - The list may be empty. A valid email address consists of characters, - an '@', and more characters. It should not contain the default contact - """ - - # YAML automatically loads list-like strings as lists. - if (isinstance(emails, list) and len(emails) >= 0): - for email in emails: - if ((not bool(re.match(EMAIL_PATTERN, email))) or (email == DEFAULT_CONTACT_EMAIL)): - return False - else: - return False - - return True - - -def check_eventbrite(eventbrite): - """ - 'eventbrite' (the Eventbrite registration key) must be 9 or more - digits. It may appear as an integer or as a string. - """ - - if isinstance(eventbrite, int): - return True - else: - return bool(re.match(EVENTBRITE_PATTERN, eventbrite)) - - -@look_for_fixme -def check_collaborative_notes(collaborative_notes): - """ - 'collaborative_notes' must be a valid URL. - """ - - return bool(re.match(URL_PATTERN, collaborative_notes)) - - -@look_for_fixme -def check_pass(value): - """ - This test always passes (it is used for 'checking' things like the - workshop address, for which no sensible validation is feasible). - """ - - return True - - -HANDLERS = { - 'layout': (True, check_layout, 'layout isn\'t "workshop"'), - - 'carpentry': (True, check_carpentry, 'carpentry isn\'t in ' + - ', '.join(CARPENTRIES)), - - 'country': (True, check_country, - 'country invalid: must use lowercase two-letter ISO code ' + - 'from ' + ', '.join(ISO_COUNTRY)), - - 'language': (False, check_language, - 'language invalid: must use lowercase two-letter ISO code' + - ' from ' + ', '.join(ISO_LANGUAGE)), - - 'humandate': (True, check_humandate, - 'humandate invalid. Please use three-letter months like ' + - '"Jan" and four-letter years like "2025"'), - - 'humantime': (True, check_humantime, - 'humantime doesn\'t include numbers'), - - 'startdate': (True, check_date, - 'startdate invalid. Must be of format year-month-day, ' + - 'i.e., 2014-01-31'), - - 'enddate': (False, check_date, - 'enddate invalid. Must be of format year-month-day, i.e.,' + - ' 2014-01-31'), - - 'latlng': (True, check_latitude_longitude, - 'latlng invalid. Check that it is two floating point ' + - 'numbers, separated by a comma'), - - 'instructor': (True, check_instructors, - 'instructor list isn\'t a valid list of format ' + - '["First instructor", "Second instructor",..]'), - - 'helper': (True, check_helpers, - 'helper list isn\'t a valid list of format ' + - '["First helper", "Second helper",..]'), - - 'email': (True, check_emails, - 'contact email list isn\'t a valid list of format ' + - '["me@example.org", "you@example.org",..] or contains incorrectly formatted email addresses or ' + - '"{0}".'.format(DEFAULT_CONTACT_EMAIL)), - - 'eventbrite': (False, check_eventbrite, 'Eventbrite key appears invalid'), - - 'collaborative_notes': (False, check_collaborative_notes, 'Collaborative Notes URL appears invalid'), - - 'venue': (False, check_pass, 'venue name not specified'), - - 'address': (False, check_pass, 'address not specified') -} - -# REQUIRED is all required categories. -REQUIRED = {k for k in HANDLERS if HANDLERS[k][0]} - -# OPTIONAL is all optional categories. -OPTIONAL = {k for k in HANDLERS if not HANDLERS[k][0]} - - -def check_blank_lines(reporter, raw): - """ - Blank lines are not allowed in category headers. - """ - - lines = [(i, x) for (i, x) in enumerate( - raw.strip().split('\n')) if not x.strip()] - reporter.check(not lines, - None, - 'Blank line(s) in header: {0}', - ', '.join(["{0}: {1}".format(i, x.rstrip()) for (i, x) in lines])) - - -def check_categories(reporter, left, right, msg): - """ - Report differences (if any) between two sets of categories. - """ - - diff = left - right - reporter.check(len(diff) == 0, - None, - '{0}: offending entries {1}', - msg, sorted(list(diff))) - - -def check_file(reporter, path, data): - """ - Get header from file, call all other functions, and check file for - validity. - """ - - # Get metadata as text and as YAML. - raw, header, body = split_metadata(path, data) - - # Do we have any blank lines in the header? - check_blank_lines(reporter, raw) - - # Look through all header entries. If the category is in the input - # file and is either required or we have actual data (as opposed to - # a commented-out entry), we check it. If it *isn't* in the header - # but is required, report an error. - for category in HANDLERS: - required, handler, message = HANDLERS[category] - if category in header: - if required or header[category]: - reporter.check(handler(header[category]), - None, - '{0}\n actual value "{1}"', - message, header[category]) - elif required: - reporter.add(None, - 'Missing mandatory key "{0}"', - category) - - # Check whether we have missing or too many categories - seen_categories = set(header.keys()) - check_categories(reporter, REQUIRED, seen_categories, - 'Missing categories') - check_categories(reporter, seen_categories, REQUIRED.union(OPTIONAL), - 'Superfluous categories') - - -def check_config(reporter, filename): - """ - Check YAML configuration file. - """ - - config = load_yaml(filename) - - kind = config.get('kind', None) - reporter.check(kind == 'workshop', - filename, - 'Missing or unknown kind of event: {0}', - kind) - - carpentry = config.get('carpentry', None) - reporter.check(carpentry in ('swc', 'dc', 'lc', 'cp'), - filename, - 'Missing or unknown carpentry: {0}', - carpentry) - - -def main(): - '''Run as the main program.''' - - if len(sys.argv) != 2: - print(USAGE, file=sys.stderr) - sys.exit(1) - - root_dir = sys.argv[1] - index_file = os.path.join(root_dir, 'index.html') - config_file = os.path.join(root_dir, '_config.yml') - - reporter = Reporter() - check_config(reporter, config_file) - check_unwanted_files(root_dir, reporter) - with open(index_file, encoding='utf-8') as reader: - data = reader.read() - check_file(reporter, index_file, data) - reporter.report() - - -if __name__ == '__main__': - main() diff --git a/code/.gitkeep b/code/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..0d1751a --- /dev/null +++ b/config.yaml @@ -0,0 +1,97 @@ +#------------------------------------------------------------ +# Values for this lesson. +#------------------------------------------------------------ + +# Which carpentry is this (swc, dc, lc, or cp)? +# swc: Software Carpentry +# dc: Data Carpentry +# lc: Library Carpentry +# cp: Carpentries (to use for instructor training for instance) +# incubator: The Carpentries Incubator +# +# This option supports custom types so lessons can be branded +# and themed with your own logo and alt-text (see `carpentry_description`) +# See https://carpentries.github.io/sandpaper-docs/editing.html#adding-a-custom-logo +carpentry: 'incubator' + +# Alt-text description of the lesson. +carpentry_description: 'Introduction to parallel programming in Chapel' + +# Overall title for pages. +title: 'Introduction to High-Performance Computing in Chapel' + +# Date the lesson was created (YYYY-MM-DD, this is empty by default) +created: ~ # FIXME + +# Comma-separated list of keywords for the lesson +keywords: 'software, data, lesson, The Carpentries, HPC, Chapel' # FIXME + +# Life cycle stage of the lesson +# possible values: pre-alpha, alpha, beta, stable +life_cycle: 'alpha' # FIXME + +# License of the lesson +license: 'CC-BY 4.0' + +# Link to the source repository for this lesson +source: 'https://github.com/carpentries/workbench-template-md' # FIXME + +# Default branch of your lesson +branch: 'main' + +# Who to contact if there are any issues +contact: 'team@carpentries.org' # FIXME + +# Navigation ------------------------------------------------ +# +# Use the following menu items to specify the order of +# individual pages in each dropdown section. Leave blank to +# include all pages in the folder. +# +# Example ------------- +# +# episodes: +# - introduction.md +# - first-steps.md +# +# learners: +# - setup.md +# +# instructors: +# - instructor-notes.md +# +# profiles: +# - one-learner.md +# - another-learner.md + +# Order of episodes in your lesson +episodes: +- 01-intro.md +- 02-variables.md +- 03-ranges-arrays.md +- 04-conditionals.md +- 05-loops.md +- 06-procedures.md +- 07-commandargs.md +- 08-timing.md +- 11-parallel-intro.md +- 12-fire-forget-tasks.md +- 13-synchronization.md +- 14-parallel-case-study.md +- 21-locales.md +- 22-domains.md +# - introduction.md + +# Information for Learners +learners: + +# Information for Instructors +instructors: + +# Learner Profiles +profiles: + +# Customisation --------------------------------------------- +# +# This space below is where custom yaml items (e.g. pinning +# sandpaper and varnish versions) should live diff --git a/data/.gitkeep b/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/_episodes/.gitkeep b/episodes/.gitkeep similarity index 100% rename from _episodes/.gitkeep rename to episodes/.gitkeep diff --git a/episodes/01-intro.md b/episodes/01-intro.md new file mode 100644 index 0000000..17237e9 --- /dev/null +++ b/episodes/01-intro.md @@ -0,0 +1,143 @@ +--- +title: "Introduction to Chapel" +teaching: 15 +exercises: 15 +--- + +:::::::::::::::::::::::::::::::::::::: questions +- "What is Chapel and why is it useful?" +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives +- "Write and execute our first chapel program." +:::::::::::::::::::::::::::::::::::::::::::::::: + +**_Chapel_** is a modern programming language, developed by _Cray Inc._, that supports HPC via high-level +abstractions for data parallelism and task parallelism. These abstractions allow the users to express parallel +codes in a natural, almost intuitive, manner. In contrast with other high-level parallel languages, however, +Chapel was designed around a _multi-resolution_ philosophy. This means that users can incrementally add more +detail to their original code prototype, to optimise it to a particular computer as closely as required. + +In a nutshell, with Chapel we can write parallel code with the simplicity and readability of scripting +languages such as Python or MATLAB, but achieving performance comparable to compiled languages like C or +Fortran (+ traditional parallel libraries such as MPI or OpenMP). + +In this lesson we will learn the basic elements and syntax of the language; then we will study **_task +parallelism_**, the first level of parallelism in Chapel, and finally we will use parallel data structures and +**_data parallelism_**, which is the higher level of abstraction, in parallel programming, offered by Chapel. + +## Getting started + +Chapel is a compilable language which means that we must **_compile_** our **_source code_** to generate a +**_binary_** or **_executable_** that we can then run in the computer. + +Chapel source code must be written in text files with the extension **_.chpl_**. Let's write a simple "hello +world"-type program to demonstrate how we write Chapel code! Using your favourite text editor, create the file +`hello.chpl` with the following content: + +```bash +writeln('If we can see this, everything works!'); +``` + +This program can then be compiled with the following bash command: + +```bash +chpl --fast hello.chpl -o hello.o +``` + +The flag `--fast` indicates the compiler to optimise the binary to run as fast as possible in the given +architecture. The `-o` option tells Chapel what to call the final output program, in this case `hello.o`. + +To run the code, you execute it as you would any other program: + +```bash +./hello.o +``` +```output +If we can see this, everything works! +``` + +## Running on a cluster + +Depending on the code, it might utilise several or even all cores on the current node. The command above +implies that you are allowed to utilise all cores. This might not be the case on an HPC cluster, where a login +node is shared by many people at the same time, and where it might not be a good idea to occupy all cores on a +login node with CPU-intensive tasks. + +On Compute Canada clusters Cedar and Graham we have two versions of Chapel, one is a single-locale +(single-node) Chapel, and the other is a multi-locale (multi-node) Chapel. For now, we will start with +single-locale Chapel. If you are logged into Cedar or Graham, you'll need to load the single-locale Chapel +module: + +```bash +module load gcc chapel-multicore +``` + +Then, for running a test code on a cluster you would submit an interactive job to the queue + +```bash +salloc --time=0:30:0 --ntasks=1 --cpus-per-task=3 --mem-per-cpu=1000 --account=def-guest +``` + +and then inside that job compile and run the test code + +```bash +chpl --fast hello.chpl -o hello.o +./hello.o +``` + +For production jobs, you would compile the code and then submit a batch script to the queue: + +```bash +chpl --fast hello.chpl -o hello.o +sbatch script.sh +``` + +where the script `script.sh` would set all Slurm variables and call the executable `mybinary`. + +## Case study + +Along all the Chapel lessons we will be using the following _case study_ as the leading thread of the +discussion. Essentially, we will be building, step by step, a Chapel code to solve the **_Heat transfer_** +problem described below. Then we will parallelize the code to improve its performance. + +Suppose that we have a square metallic plate with some initial heat distribution or **_initial +conditions_**. We want to simulate the evolution of the temperature across the plate when its border is in +contact with a different heat distribution that we call the **_boundary conditions_**. + +The Laplace equation is the mathematical model for the evolution of the temperature in the plate. To solve +this equation numerically, we need to **_discretise_** it, i.e. to consider the plate as a grid, or matrix of +points, and to evaluate the temperature on each point at each iteration, according to the following +**_difference equation_**: + +```chpl +T[i,j] = 0.25 (Tp[i-1,j] + Tp[i+1,j] + Tp[i,j-1] + Tp[i,j+1]) +``` + +Here `T` stands for the temperature at the current iteration, while `Tp` contains the temperature calculated +at the past iteration (or the initial conditions in case we are at the first iteration). The indices `i` and +`j` indicate that we are working on the point of the grid located at the *i*th row and the *j*th column. + +So, our objective is to: + +> ## Goals +> 1. Write a code to implement the difference equation above.The code should +> have the following requirements: +> +> * It should work for any given number of rows and columns in the grid. +> * It should run for a given number of iterations, or until the difference +> between `T` and `Tp` is smaller than a given tolerance value. +> * It should output the temperature at a desired position on the grid every +> given number of iterations. +> +> 2. Use task parallelism to improve the performance of the code and run it in +> the cluster +> 3. Use data parallelism to improve the performance of the code and run it in +> the cluster. +{: .checklist} + +::::::::::::::::::::::::::::::::::::: keypoints +- "Chapel is a compiled language - any programs we make must be compiled with `chpl`." +- "The `--fast` flag instructs the Chapel compiler to optimise our code." +- "The `-o` flag tells the compiler what to name our output (otherwise it gets named `a.out`)" +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/02-variables.md b/episodes/02-variables.md similarity index 58% rename from _episodes/02-variables.md rename to episodes/02-variables.md index 87ca04a..a136b1a 100644 --- a/_episodes/02-variables.md +++ b/episodes/02-variables.md @@ -2,24 +2,23 @@ title: "Basic syntax and variables" teaching: 15 exercises: 15 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "How do I write basic Chapel code?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "Perform basic maths in Chapel." - "Understand Chapel's basic data types." - "Understand how to read and fix errors." - "Know how to define and use data stored as variables." -keypoints: -- "A comment is preceded with `//` or surrounded by `/* and `*/`" -- "All variables hold a certain type of data." -- "Using `const` instead of `var` prevents reassignment." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: -Basic maths in Chapel works the same as other programming languages. Try -compiling the following code to see how the different mathematical operators -work. +Basic maths in Chapel works the same as other programming languages. Try compiling the following code to see +how the different mathematical operators work. -``` +```chpl writeln(4 + 5); writeln(4 - 5); writeln(4 * 5); @@ -27,20 +26,17 @@ writeln(4 / 5); // integer division writeln(4.0 / 5.0); // normal division writeln(4 ** 5); // exponentiation ``` -{: .bash} -In this example, our code is called `operators.chpl`. You can compile it with -the following commands: +In this example, our code is called `operators.chpl`. You can compile it with the following commands: -``` +```bash chpl operators.chpl --fast -o operators.o ./operators.o ``` -{: .bash} You should see output that looks something like the following: -``` +```output 9 -1 20 @@ -48,45 +44,37 @@ You should see output that looks something like the following: 0.8 1024 ``` -{: .output} -Code beginning with `//` is interpreted as a comment — it does not get -run. Comments are very valuable when writing code, because they allow us to -write notes to ourselves about what each piece of code does. You can also -create block comments with `/*` and `*/`: +Code beginning with `//` is interpreted as a comment — it does not get run. Comments are very valuable +when writing code, because they allow us to write notes to ourselves about what each piece of code does. You +can also create block comments with `/*` and `*/`: -``` +```chpl /* This is a block comment. It can span as many lines as you want! (like this) */ ``` -{: .source} ## Variables -Granted, we probably want to do more than basic maths with Chapel. We will need -to store the results of complex operations using variables. Variables in -programming are not the same as the mathematical concept. In programming, a -variable is an allocated space in the memory of the computer, where we can -store information or data while executing a program. A variable has three -elements: +Granted, we probably want to do more than basic maths with Chapel. We will need to store the results of +complex operations using variables. Variables in programming are not the same as the mathematical concept. In +programming, a variable is an allocated space in the memory of the computer, where we can store information or +data while executing a program. A variable has three elements: 1. a **_name_** or label, to identify the variable 2. a **_type_**, that indicates the kind of data that we can store in it, and 3. a **_value_**, the actual information or data stored in the variable. -When we store a value in a variable for the first time, we say that we -**_initialised_** it. Further changes to the value of a variable are called -**_assignments_**, in general, `x=a` means that we assign the value *a* to the -variable *x*. +When we store a value in a variable for the first time, we say that we **_initialised_** it. Further changes +to the value of a variable are called **_assignments_**, in general, `x=a` means that we assign the value *a* +to the variable *x*. -Variables in Chapel are declared with the `var` or `const` keywords. When a -variable declared as const is initialised, its value cannot be modified anymore -during the execution of the program. +Variables in Chapel are declared with the `var` or `const` keywords. When a variable declared as const is +initialised, its value cannot be modified anymore during the execution of the program. -In Chapel, to declare a variable we must specify the type of the variable, or -initialise it in place with some value. The common variable types in Chapel -are: +In Chapel, to declare a variable we must specify the type of the variable, or initialise it in place with some +value. The common variable types in Chapel are: * integer `int` (positive or negative whole numbers) * floating-point number `real` (decimal values) @@ -97,120 +85,104 @@ If a variable is declared without a type, Chapel will infer it from the given initial value. We can use the stored variable simply by using its name anywhere in our code (called `variables.chpl`). -~~~ +```chpl const test = 100; writeln('The value of test is: ', test); writeln(test / 4); -~~~ -{: .source} - ``` + +```bash chpl variables.chpl -o variables.o --fast ./variables.o ``` -{: .bash} -``` + +```output The value of test is: 100 25 ``` -{: .output} -This constant variable `test` will be created as an integer, and initialised -with the value 100. No other values can be assigned to these variables during -the execution of the program. What happens if we try to modify a constant -variable like `test`? +This constant variable `test` will be created as an integer, and initialised with the value 100. No other +values can be assigned to these variables during the execution of the program. What happens if we try to +modify a constant variable like `test`? -~~~ +```chpl const test = 100; test = 200; writeln('The value of test is: ', test); writeln(test / 4); -~~~ -{: .source} ``` + +```bash chpl variables.chpl -o variables.o ``` -{: .bash} -``` + +```error variables.chpl:2: error: illegal lvalue in assignment ``` -{: .error} -The compiler threw an error, and did not compile our program. This is a feature -of compiled languages - if there is something wrong, we will typically see an -error while writing our program, instead of while running it. Although we -already kind of know why the error was caused (we tried to reassign the value -of a `const` variable, which by definition cannot be changed), let's walk -through the error as an example of how to troubleshoot our programs. +The compiler threw an error, and did not compile our program. This is a feature of compiled languages - if +there is something wrong, we will typically see an error while writing our program, instead of while running +it. Although we already kind of know why the error was caused (we tried to reassign the value of a `const` +variable, which by definition cannot be changed), let's walk through the error as an example of how to +troubleshoot our programs. -* `variables.chpl:2:` indicates that the error was caused on line 2 of our - `variables.chpl` file. +* `variables.chpl:2:` indicates that the error was caused on line 2 of our `variables.chpl` file. -* `error:` indicates that the issue was an error, and blocks compilation. - Sometimes the compiler will just give us warning or information, not - necessarily errors. When we see something that is not an error, we should - carefully read the output and consider if it necessitates changing our code. - Errors must be fixed, as they will block the code from compiling. +* `error:` indicates that the issue was an error, and blocks compilation. Sometimes the compiler will just + give us warning or information, not necessarily errors. When we see something that is not an error, we + should carefully read the output and consider if it necessitates changing our code. Errors must be fixed, + as they will block the code from compiling. -* `illegal lvalue in assignment` indicates that the left hand side of our - assignment expression (`lvalue`) was illegal. We were trying to reassign a - `const` variable, which is explicitly not allowed in Chapel. +* `illegal lvalue in assignment` indicates that the left hand side of our assignment expression (`lvalue`) was + illegal. We were trying to reassign a `const` variable, which is explicitly not allowed in Chapel. -To fix this error, we can change `const` to `var` when declaring our `test` -variable. `var` indicates a variable that can be reassigned. +To fix this error, we can change `const` to `var` when declaring our `test` variable. `var` indicates a +variable that can be reassigned. -~~~ +```chpl var test = 100; test = 200; writeln('The value of test is: ', test); writeln(test / 4); -~~~ -{: .source} - ``` + +```bash chpl variables.chpl -o variables.o ``` -{: .bash} -``` + +```output The value of test is: 200 50 ``` -{: .output} -It worked! Now we know both how to set, use, and change a variable, as well as -the implications of using `var` and `const`. We also know how to read and -interpret errors. +It worked! Now we know both how to set, use, and change a variable, as well as the implications of using `var` +and `const`. We also know how to read and interpret errors. ## Uninitialised variables -On the other hand, if a variable is declared without an initial value, Chapel -will initialise it with a default value depending on the declared type (0.0 for -real variables, for example). The following variables will be created as real -floating point numbers equal to 0.0. +On the other hand, if a variable is declared without an initial value, Chapel will initialise it with a +default value depending on the declared type (0.0 for real variables, for example). The following variables +will be created as real floating point numbers equal to 0.0. -~~~ +```chpl var curdif: real; //here we will store the greatest difference in temperature from one iteration to another var tt: real; //for temporary results when computing the temperatures -~~~ -{: .source} +``` -Of course, we can use both, the initial value and the type, when declaring a -variable as follows: +Of course, we can use both, the initial value and the type, when declaring a variable as follows: -~~~ +```chpl const mindif=0.0001: real; //smallest difference in temperature that would be accepted before stopping const n=20: int; //the temperature at the desired position will be printed every n iterations -~~~ -{: .source} +``` *This is not necessary, but it could help to make the code more readable.* -Let's practice defining variables and use this as the starting point of our -simulation code. In these examples, our simulation will be in the file -`base_solution.chpl`. +Let's practice defining variables and use this as the starting point of our simulation code. In these +examples, our simulation will be in the file `base_solution.chpl`. -``` +```chpl const rows = 100; // number of rows in matrix const cols = 100; // number of columns in matrix const niter = 500; // number of iterations @@ -221,4 +193,9 @@ var tt: real; // for temporary results when computing the temp const mindif = 0.0001: real; // smallest difference in temperature that would be accepted before stopping const n = 20: int; // the temperature at the desired position will be printed every n iterations ``` -{: .source} + +::::::::::::::::::::::::::::::::::::: keypoints +- "A comment is preceded with `//` or surrounded by `/* and `*/`" +- "All variables hold a certain type of data." +- "Using `const` instead of `var` prevents reassignment." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md similarity index 66% rename from _episodes/03-ranges-arrays.md rename to episodes/03-ranges-arrays.md index 369c349..f12585d 100644 --- a/_episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -2,58 +2,52 @@ title: "Ranges and arrays" teaching: 60 exercises: 30 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "What is Chapel and why is it useful?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "Learn to define and use ranges and arrays." -keypoints: -- "A range is a sequence of integers." -- "An array holds a sequence of values." -- "Chapel arrays can start at any index, not just 0 or 1." -- "You can index arrays with the `[]` brackets." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: ## Ranges and Arrays -A series of integers (1,2,3,4,5, for example), is called a **_range_**. Ranges -are generated with the `..` operator, and are useful, among other things, to -declare **_arrays_** of variables. +A series of integers (1,2,3,4,5, for example), is called a **_range_**. Ranges are generated with the `..` +operator, and are useful, among other things, to declare **_arrays_** of variables. Let's examine what a range looks like (`ranges.chpl` in this example): -``` +```chpl var example_range = 0..10; writeln('Our example range was set to: ', example_range); ``` -{: .source} -``` + +```bash chpl ranges.chpl -o ranges.o ./ranges.o ``` -{: .bash} -``` + +```output Our example range was set to: 0..10 ``` -{: .output} -An array is a multidimensional sequence of values. Arrays can be any size, and -are defined using ranges: Let's define a 1-dimensional array of the size -`example_range` and see what it looks like. Notice how the size of an array is -included with its type. +An array is a multidimensional sequence of values. Arrays can be any size, and are defined using ranges: Let's +define a 1-dimensional array of the size `example_range` and see what it looks like. Notice how the size of an +array is included with its type. -``` +```chpl var example_range = 0..10; writeln('Our example range was set to: ', example_range); var example_array: [example_range] real; writeln('Our example array is now: ', example_array); ``` -{: .source} -We can reassign the values in our example array the same way we would reassign -a variable. An array can either be set all to a single value, or a sequence of -values. +We can reassign the values in our example array the same way we would reassign a variable. An array can either +be set all to a single value, or a sequence of values. -``` +```chpl var example_range = 0..10; writeln('Our example range was set to: ', example_range); var example_array: [example_range] real; @@ -63,32 +57,28 @@ writeln('When set to 5: ', example_array); example_array = 1..11; writeln('When set to a range: ', example_array); ``` -{: .source} -``` +```bash chpl ranges.chpl -o ranges.o ./ranges.o ``` -{: .bash} -``` + +```output Our example range was set to: 0..10 Our example array is now: 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 When set to 5: 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 When set to a range: 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 10.0 11.0 ``` -{: .output} -Notice how ranges are "right inclusive", the last number of a range is included -in the range. This is different from languages like Python where this does not -happen. +Notice how ranges are "right inclusive", the last number of a range is included in the range. This is +different from languages like Python where this does not happen. ## Indexing elements -One final thing - we can retrieve and reset specific values of an array using -`[]` notation. Let's try retrieving and setting a specific value in our example -so far: +One final thing - we can retrieve and reset specific values of an array using `[]` notation. Let's try +retrieving and setting a specific value in our example so far: -``` +```chpl var example_range = 0..10; writeln('Our example range was set to: ', example_range); var example_array: [example_range] real; @@ -103,14 +93,13 @@ writeln(example_array[5]); example_array[5] = 99999; writeln(example_array); ``` -{: .source} -``` +```bash chpl ranges.chpl -o ranges.o ./ranges.o ``` -{: .bash} -``` + +```output Our example range was set to: 0..10 Our example array is now: 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 When set to 5: 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 @@ -118,16 +107,13 @@ When set to a range: 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 10.0 11.0 6.0 1.0 2.0 3.0 4.0 5.0 99999.0 7.0 8.0 9.0 10.0 11.0 ``` -{: .output} -One very important thing to note - in this case, index 5 was actually the 6th -element. This was caused by how we set up our array. When we defined our array -using a range starting at 0, element 5 corresponds to the 6th element. Unlike -most other programming languages, arrays in Chapel do not start at a fixed -value - they can start at any number depending on how we define them! For -instance, let's redefine example_range to start at 5: +One very important thing to note - in this case, index 5 was actually the 6th element. This was caused by how +we set up our array. When we defined our array using a range starting at 0, element 5 corresponds to the 6th +element. Unlike most other programming languages, arrays in Chapel do not start at a fixed value - they can +start at any number depending on how we define them! For instance, let's redefine example_range to start at 5: -``` +```chpl var example_range = 5..15; writeln('Our example range was set to: ', example_range); var example_array: [example_range] real; @@ -142,14 +128,13 @@ writeln(example_array[5]); example_array[5] = 99999; writeln(example_array); ``` -{: .source} -``` +```bash chpl ranges.chpl -o ranges.o ./ranges.o ``` -{: .bash} -``` + +```output Our example range was set to: 5..15 Our example array is now: 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 When set to 5: 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 5.0 @@ -157,33 +142,27 @@ When set to a range: 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 10.0 11.0 1.0 99999.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 10.0 11.0 ``` -{: .output} ## Back to our simulation Let's define a two dimensional array for use in our simulation: -~~~ +```chpl // this is our "plate" var temp: [0..rows+1, 0..cols+1] real = 25; -~~~ -{:.source} - -This is a matrix (2D array) with (`rows + 2`) rows and (`cols + 2`) columns of -real numbers, all initialised as 25.0. The ranges `0..rows+1` and `0..cols+1` -used here, not only define the size and shape of the array, they stand for the -indices with which we could access particular elements of the array using the -`[ , ]` notation. For example, `temp[0,0]` is the real variable located at the -first row and first column of the array `temp`, while `temp[3,7]` is the one at -the 4th row and 8th column; `temp[2,3..15]` access columns 4th to 16th of the -3th row of `temp`, and `temp[0..3,4]` corresponds to the first 4 rows on the -5th column of `temp`. - -We must now be ready to start coding our simulations. Let's print some -information about the initial configuration, compile the code, and execute it -to see if everything is working as expected. - -~~~ +``` + +This is a matrix (2D array) with (`rows + 2`) rows and (`cols + 2`) columns of real numbers, all initialised +as 25.0. The ranges `0..rows+1` and `0..cols+1` used here, not only define the size and shape of the array, +they stand for the indices with which we could access particular elements of the array using the `[ , ]` +notation. For example, `temp[0,0]` is the real variable located at the first row and first column of the array +`temp`, while `temp[3,7]` is the one at the 4th row and 8th column; `temp[2,3..15]` access columns 4th to 16th +of the 3th row of `temp`, and `temp[0..3,4]` corresponds to the first 4 rows on the 5th column of `temp`. + +We must now be ready to start coding our simulations. Let's print some information about the initial +configuration, compile the code, and execute it to see if everything is working as expected. + +```chpl const rows = 100; const cols = 100; const niter = 500; @@ -196,15 +175,21 @@ var temp: [0..rows+1, 0..cols+1] real = 25; writeln('This simulation will consider a matrix of ', rows, ' by ', cols, ' elements.'); writeln('Temperature at start is: ', temp[x, y]); -~~~ -{:.source} -~~~ +``` + +```bash >> chpl base_solution.chpl -o base_solution >> ./base_solution -~~~ -{:.bash} -~~~ +``` + +```output This simulation will consider a matrix of 100 by 100 elements. Temperature at start is: 25.0 -~~~ -{:.output} +``` + +::::::::::::::::::::::::::::::::::::: keypoints +- "A range is a sequence of integers." +- "An array holds a sequence of values." +- "Chapel arrays can start at any index, not just 0 or 1." +- "You can index arrays with the `[]` brackets." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/04-conditionals.md b/episodes/04-conditionals.md similarity index 62% rename from _episodes/04-conditionals.md rename to episodes/04-conditionals.md index 4927527..d1d2507 100644 --- a/_episodes/04-conditionals.md +++ b/episodes/04-conditionals.md @@ -2,21 +2,22 @@ title: "Conditional statements" teaching: 60 exercises: 30 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "How do I add conditional logic to my code?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "You can use the `==`, `>`, `>=`, etc. operators to make a comparison that returns true or false." -keypoints: -- "Conditional statements in Chapel are very similar to these in other languages." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: -Chapel, as most *high level programming languages*, has different statements to -control the flow of the program or code. The conditional statements are: the -**_if statement_**, and the **_while statement_**. These statements both rely -on comparisons between values. Let's try a few comparisons to see how they work +Chapel, as most *high level programming languages*, has different statements to control the flow of the +program or code. The conditional statements are: the **_if statement_**, and the **_while statement_**. These +statements both rely on comparisons between values. Let's try a few comparisons to see how they work (`conditionals.chpl`): -``` +```chpl writeln(1 == 2); writeln(1 != 2); writeln(1 > 2); @@ -24,14 +25,13 @@ writeln(1 >= 2); writeln(1 < 2); writeln(1 <= 2); ``` -{: .source} -``` +```bash chpl conditionals.chpl -o conditionals.o ./conditionals.o ``` -{: .bash} -``` + +```output false true false @@ -39,13 +39,11 @@ false true true ``` -{: .output} -You can combine comparisons with the `&&` (AND) and `||` (OR) operators. `&&` -only returns `true` if both conditions are true, while `||` returns `true` if -either condition is true. +You can combine comparisons with the `&&` (AND) and `||` (OR) operators. `&&` only returns `true` if both +conditions are true, while `||` returns `true` if either condition is true. -``` +```chpl writeln(1 == 2); writeln(1 != 2); writeln(1 > 2); @@ -56,13 +54,13 @@ writeln(true && true); writeln(true && false); writeln(true || false); ``` -{: .source} -``` + +```bash chpl conditionals.chpl -o conditionals.o ./conditionals.o ``` -{: .bash} -``` + +```output false true false @@ -73,27 +71,23 @@ true false true ``` -{: .output} ## Control flow The general syntax of a while statement is: -``` +```chpl while condition do {instructions} ``` -{: .source} -The code flows as follows: first, the condition is evaluated, and then, if it -is satisfied, all the instructions within the curly brackets are executed one -by one. This will be repeated over and over again until the condition does not -hold anymore. +The code flows as follows: first, the condition is evaluated, and then, if it is satisfied, all the +instructions within the curly brackets are executed one by one. This will be repeated over and over again +until the condition does not hold anymore. -The main loop in our simulation can be programmed using a while statement like -this +The main loop in our simulation can be programmed using a while statement like this -~~~ +```chpl //this is the main loop of the simulation var c = 0; var curdif = mindif; @@ -102,57 +96,49 @@ while (c < niter && curdif >= mindif) do c += 1; // actual simulation calculations will go here } -~~~ -{:.source} - -Essentially, what we want is to repeat all the code inside the curly brackets -until the number of iterations is greater than or equal to `niter`, or the -difference of temperature between iterations is less than `mindif`. (Note that -in our case, as `curdif` was not initialised when declared -and thus Chapel -assigned it the default real value 0.0-, we need to assign it a value greater -than or equal to 0.001, or otherwise the condition of the while statement will -never be satisfied. A good starting point is to simple say that `curdif` is -equal to `mindif`). - -To count iterations we just need to keep adding 1 to the counter variable `c`. -We could do this with `c=c+1`, or with the compound assignment, `+=`, as in the -code above. To program the rest of the logic inside the curly brackets, on the -other hand, we will need more elaborated instructions. - -Let's focus, first, on printing the temperature every 20 iterations. To achieve -this, we only need to check whether `c` is a multiple of 20, and in that case, -to print the temperature at the desired position. This is the type of control -that an **_if statement_** give us. The general syntax is: - ``` + +Essentially, what we want is to repeat all the code inside the curly brackets until the number of iterations +is greater than or equal to `niter`, or the difference of temperature between iterations is less than +`mindif`. (Note that in our case, as `curdif` was not initialised when declared -and thus Chapel assigned it +the default real value 0.0-, we need to assign it a value greater than or equal to 0.001, or otherwise the +condition of the while statement will never be satisfied. A good starting point is to simple say that `curdif` +is equal to `mindif`). + +To count iterations we just need to keep adding 1 to the counter variable `c`. We could do this with `c=c+1`, +or with the compound assignment, `+=`, as in the code above. To program the rest of the logic inside the curly +brackets, on the other hand, we will need more elaborated instructions. + +Let's focus, first, on printing the temperature every 20 iterations. To achieve this, we only need to check +whether `c` is a multiple of 20, and in that case, to print the temperature at the desired position. This is +the type of control that an **_if statement_** give us. The general syntax is: + +```chpl if condition then {instructions A} else {instructions B} ``` -{: .source} -The set of instructions A is executed once if the condition is satisfied; the -set of instructions B is executed otherwise (the else part of the if statement -is optional). +The set of instructions A is executed once if the condition is satisfied; the set of instructions B is +executed otherwise (the else part of the if statement is optional). So, in our case this would do the trick: -~~~ +```chpl if (c % 20 == 0) { writeln('Temperature at iteration ', c, ': ', temp[x, y]); } -~~~ -{:.source} +``` -Note that when only one instruction will be executed, there is no need to use -the curly brackets. `%` is the modulo operator, it returns the remainder after -the division (i.e. it returns zero when `c` is multiple of 20). +Note that when only one instruction will be executed, there is no need to use the curly brackets. `%` is the +modulo operator, it returns the remainder after the division (i.e. it returns zero when `c` is multiple of +20). Let's compile and execute our code to see what we get until now -``` +```chpl const rows = 100; const cols = 100; const niter = 500; @@ -178,13 +164,13 @@ while (c < niter) do } } ``` -{: .source} -~~~ + +```bash chpl base_solution.chpl -o base_solution.o ./base_solution.o -~~~ -{: .bash} ``` + +```output This simulation will consider a matrix of 100 by 100 elements. Temperature at start is: 25.0 Temperature at iteration 20: 25.0 @@ -213,7 +199,10 @@ Temperature at iteration 460: 25.0 Temperature at iteration 480: 25.0 Temperature at iteration 500: 25.0 ``` -{:.output} -Of course the temperature is always 25.0 at any iteration other than the -initial one, as we haven't done any computation yet. +Of course the temperature is always 25.0 at any iteration other than the initial one, as we haven't done any +computation yet. + +::::::::::::::::::::::::::::::::::::: keypoints +- "Conditional statements in Chapel are very similar to these in other languages." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/episodes/05-loops.md b/episodes/05-loops.md new file mode 100644 index 0000000..f967234 --- /dev/null +++ b/episodes/05-loops.md @@ -0,0 +1,331 @@ +--- +title: "Getting started with loops" +teaching: 60 +exercises: 30 +--- + +:::::::::::::::::::::::::::::::::::::: questions +- "How do I get run the same piece of code repeatedly?" +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives +- "First objective." +:::::::::::::::::::::::::::::::::::::::::::::::: + +To compute the current temperature of an element of `temp`, we need to add all the surrounding elements in +`past_temp`, and divide the result by 4. And, essentially, we need to repeat this process for all the elements +of `temp`, or, in other words, we need to *iterate* over the elements of `temp`. When it comes to iterate over +a given number of elements, the **_for-loop_** is what we want to use. The for-loop has the following general +syntax: + +```chpl +for index in iterand do +{instructions} +``` + +The *iterand* is a function or statement that expresses an iteration; it could be the range 1..15, for +example. *index* is a variable that exists only in the context of the for-loop, and that will be taking the +different values yielded by the iterand. The code flows as follows: index takes the first value yielded by the +iterand, and keeps it until all the instructions inside the curly brackets are executed one by one; then, +index takes the second value yielded by the iterand, and keeps it until all the instructions are executed +again. This pattern is repeated until index takes all the different values expressed by the iterand. + +This for loop, for example + +```chpl +// calculate the new current temperatures (temp) using the past temperatures (past_temp) +for i in 1..rows do +{ + // do this for every row +} +``` + +will allow us to iterate over the rows of `temp`. Now, for each row we also need to iterate over all the +columns in order to access every single element of `temp`. This can be done with nested for loops like this + +```chpl +// calculate the new current temperatures (temp) using the past temperatures (past_temp) +for i in 1..rows do +{ + // do this for every row + for j in 1..cols do + { + // and this for every column in the row i + } +} +``` + +Now, inside the inner loop, we can use the indices `i` and `j` to perform the required computations as +follows: + +```chpl +// calculate the new current temperatures (temp) using the past temperatures (past_temp) +for i in 1..rows do +{ + // do this for every row + for j in 1..cols do + { + // and this for every column in the row i + temp[i,j]=(past_temp[i-1,j]+past_temp[i+1,j]+past_temp[i,j-1]+past_temp[i,j+1])/4; + } +} +past_temp=temp; +``` + +Note that at the end of the outer for-loop, when all the elements in `temp` are already calculated, we update +`past_temp` with the values of `temp`; this way everything is set up for the next iteration of the main while +statement. + +Now let's compile and execute our code again: + +```bash +>> chpl base_solution.chpl -o base_solution +>> ./base_solution +``` + +```output +The simulation will consider a matrix of 100 by 100 elements, +it will run up to 500 iterations, or until the largest difference +in temperature between iterations is less than 0.0001. +You are interested in the evolution of the temperature at the +position (50,50) of the matrix... + +and here we go... +Temperature at iteration 0: 25.0 +Temperature at iteration 20: 25.0 +Temperature at iteration 40: 25.0 +Temperature at iteration 60: 25.0 +Temperature at iteration 80: 25.0 +Temperature at iteration 100: 25.0 +Temperature at iteration 120: 25.0 +Temperature at iteration 140: 25.0 +Temperature at iteration 160: 25.0 +Temperature at iteration 180: 25.0 +Temperature at iteration 200: 25.0 +Temperature at iteration 220: 24.9999 +Temperature at iteration 240: 24.9996 +Temperature at iteration 260: 24.9991 +Temperature at iteration 280: 24.9981 +Temperature at iteration 300: 24.9963 +Temperature at iteration 320: 24.9935 +Temperature at iteration 340: 24.9893 +Temperature at iteration 360: 24.9833 +Temperature at iteration 380: 24.9752 +Temperature at iteration 400: 24.9644 +Temperature at iteration 420: 24.9507 +Temperature at iteration 440: 24.9337 +Temperature at iteration 460: 24.913 +Temperature at iteration 480: 24.8883 +Temperature at iteration 500: 24.8595 +``` + +As we can see, the temperature in the middle of the plate (position 50,50) is slowly decreasing as the plate +is cooling down. + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 1: Can you do it? + +What would be the temperature at the top right corner of the plate? The border of the plate is in contact with +the boundary conditions, which are set to zero, so we expect the temperature at these points to decrease +faster. Modify the code to see the temperature at the top right corner. + +:::::::::::::::::::::::: solution + +To see the evolution of the temperature at the top right corner of the plate, we just need to modify `x` and +`y`. This corner correspond to the first row (`x=1`) and the last column (`y=cols`) of the plate. + +```bash +>> chpl base_solution.chpl -o base_solution +>> ./base_solution +``` + +```output +The simulation will consider a matrix of 100 by 100 elements, +it will run up to 500 iterations, or until the largest difference +in temperature between iterations is less than 0.0001. +You are interested in the evolution of the temperature at the position (1,100) of the matrix... + +and here we go... +Temperature at iteration 0: 25.0 +Temperature at iteration 20: 1.48171 +Temperature at iteration 40: 0.767179 +... +Temperature at iteration 460: 0.068973 +Temperature at iteration 480: 0.0661081 +Temperature at iteration 500: 0.0634717 +``` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 2: Can you do it? + +Now let's have some more interesting boundary conditions. Suppose that the plate is heated by a source of 80 +degrees located at the bottom right corner, and that the temperature on the rest of the border decreases +linearly as one gets farther form the corner (see the image below). Utilise for loops to setup the described +boundary conditions. Compile and run your code to see how the temperature is changing now. + +:::::::::::::::::::::::: solution + +To get the linear distribution, the 80 degrees must be divided by the number of rows or columns in our +plate. So, the following couple of for loops will give us what we want: + +```chpl +// this setup the boundary conditions +for i in 1..rows do +{ + past_temp[i,cols+1]=i*80.0/rows; + temp[i,cols+1]=i*80.0/rows; +} +for j in 1..cols do +{ + past_temp[rows+1,j]=j*80.0/cols; + temp[rows+1,j]=j*80.0/cols; +} +``` + +Note that the boundary conditions must be set in both arrays, `past_temp` and `temp`, otherwise, they will be +set to zero again after the first iteration. Also note that 80 degrees are written as a real numbe r 80.0. +The division of integers in Chapel returns an integer, then, as `rows` and `cols` are integers, we must have +80 as real so that the quotient is not truncated. + +```bash +>> chpl base_solution.chpl -o base_solution +>> ./base_solution +``` + +```output +The simulation will consider a matrix of 100 by 100 elements, it will run +up to 500 iterations, or until the largest difference in temperature +between iterations is less than 0.0001. You are interested in the evolution +of the temperature at the position (1,100) of the matrix... + +and here we go... +Temperature at iteration 0: 25.0 +Temperature at iteration 20: 2.0859 +Temperature at iteration 40: 1.42663 +... +Temperature at iteration 460: 0.826941 +Temperature at iteration 480: 0.824959 +Temperature at iteration 500: 0.823152 +``` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 3: Can you do it? + +So far, `curdif` has been always equal to `mindif`, which means that our main while loop will always run the +500 iterations. So let's update `curdif` after each iteration. Use what we have studied so far to write the +required piece of code. + +:::::::::::::::::::::::: solution + +The idea is simple, after each iteration of the while loop, we must compare all elements of `temp` and +`past_temp`, find the greatest difference, and update `curdif` with that value. The next nested for loops do +the job: + +```chpl +// update curdif, the greatest difference between temp and past_temp +curdif=0; +for i in 1..rows do +{ + for j in 1..cols do + { + tt=abs(temp[i,j]-past_temp[i,j]); + if tt>curdif then curdif=tt; + } +} +``` + +Clearly there is no need to keep the difference at every single position in the array, we just need to update +`curdif` if we find a greater one. + +```bash +>> chpl base_solution.chpl -o base_solution +>> ./base_solution +``` + +```output +The simulation will consider a matrix of 100 by 100 elements, +it will run up to 500 iterations, or until the largest difference +in temperature between iterations is less than 0.0001. +You are interested in the evolution of the temperature at the +position (1,100) of the matrix... + +and here we go... +Temperature at iteration 0: 25.0 +Temperature at iteration 20: 2.0859 +Temperature at iteration 40: 1.42663 +... +Temperature at iteration 460: 0.826941 +Temperature at iteration 480: 0.824959 +Temperature at iteration 500: 0.823152 +``` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +Now, after Exercise 3 we should have a working program to simulate our heat +transfer equation. Let's just print some additional useful information, + +```chpl +// print final information +writeln('\nFinal temperature at the desired position after ',c,' iterations is: ',temp[x,y]); +writeln('The difference in temperatures between the last two iterations was: ',curdif,'\n'); +``` + +and compile and execute our final code, + +```bash +>> chpl base_solution.chpl -o base_solution +>> ./base_solution +``` + +```output +The simulation will consider a matrix of 100 by 100 elements, +it will run up to 500 iterations, or until the largest difference +in temperature between iterations is less than 0.0001. +You are interested in the evolution of the temperature at the +position (1,100) of the matrix... + +and here we go... +Temperature at iteration 0: 25.0 +Temperature at iteration 20: 2.0859 +Temperature at iteration 40: 1.42663 +Temperature at iteration 60: 1.20229 +Temperature at iteration 80: 1.09044 +Temperature at iteration 100: 1.02391 +Temperature at iteration 120: 0.980011 +Temperature at iteration 140: 0.949004 +Temperature at iteration 160: 0.926011 +Temperature at iteration 180: 0.908328 +Temperature at iteration 200: 0.894339 +Temperature at iteration 220: 0.88302 +Temperature at iteration 240: 0.873688 +Temperature at iteration 260: 0.865876 +Temperature at iteration 280: 0.85925 +Temperature at iteration 300: 0.853567 +Temperature at iteration 320: 0.848644 +Temperature at iteration 340: 0.844343 +Temperature at iteration 360: 0.840559 +Temperature at iteration 380: 0.837205 +Temperature at iteration 400: 0.834216 +Temperature at iteration 420: 0.831537 +Temperature at iteration 440: 0.829124 +Temperature at iteration 460: 0.826941 +Temperature at iteration 480: 0.824959 +Temperature at iteration 500: 0.823152 + +Final temperature at the desired position after 500 iterations is: 0.823152 +The greatest difference in temperatures between the last two iterations was: 0.0258874 +``` + +::::::::::::::::::::::::::::::::::::: keypoints +- "Use `for` statement to organise a loop." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/06-procedures.md b/episodes/06-procedures.md similarity index 65% rename from _episodes/06-procedures.md rename to episodes/06-procedures.md index 5736211..762e13a 100644 --- a/_episodes/06-procedures.md +++ b/episodes/06-procedures.md @@ -2,66 +2,65 @@ title: "Procedures for functional programming" teaching: 15 exercises: 0 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "How do I write functions?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "Be able to write our own procedures." -keypoints: -- "Functions in Chapel are called procedures." -- "Procedures can be recursive." -- "Procedures can take a variable number of parameters." -- "Procedures can have default parameter values." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: -Similar to other programming languages, Chapel lets you define your own -functions. These are called 'procedures' in Chapel and have an -easy-to-understand syntax: +Similar to other programming languages, Chapel lets you define your own functions. These are called +'procedures' in Chapel and have an easy-to-understand syntax: -~~~ +```chpl proc addOne(n) { // n is an.bash parameter return n + 1; } writeln(addOne(10)); -~~~ -{: .source} +``` Procedures can be recursive: -~~~ +```chpl proc fibonacci(n: int): int { if n <= 1 then return n; return fibonacci(n-1) + fibonacci(n-2); } writeln(fibonacci(10)); -~~~ -{: .source} +``` They can take a variable number of parameters: -~~~ +```chpl proc maxOf(x ...?k) { // take a tuple of one type with k elements var maximum = x[1]; for i in 2..k do maximum = if maximum < x[i] then x[i] else maximum; return maximum; } writeln(maxOf(1, -5, 123, 85, -17, 3)); -~~~ -{: .source} +``` Procedures can have default parameter values: -~~~ -proc returnTuple(x: int, y: real = 3.1415926): (int,real) { // +```chpl +proc returnTuple(x: int, y: real = 3.1415926): (int,real) { return (x,y); } writeln(returnTuple(1)); writeln(returnTuple(x=2)); writeln(returnTuple(x=-10, y=10)); writeln(returnTuple(y=-1, x=3)); // the parameters can be named out of order -~~~ -{: .source} +``` -Chapel procedures have many other useful features, however, they are not -essential for learning task and data parallelism, so we refer the interested -readers to the official Chapel documentation. +Chapel procedures have many other useful features, however, they are not essential for learning task and data +parallelism, so we refer the interested readers to the official Chapel documentation. +::::::::::::::::::::::::::::::::::::: keypoints +- "Functions in Chapel are called procedures." +- "Procedures can be recursive." +- "Procedures can take a variable number of parameters." +- "Procedures can have default parameter values." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/episodes/07-commandargs.md b/episodes/07-commandargs.md new file mode 100644 index 0000000..782eeec --- /dev/null +++ b/episodes/07-commandargs.md @@ -0,0 +1,101 @@ +--- +title: "Using command-line arguments" +teaching: 60 +exercises: 30 +--- + +:::::::::::::::::::::::::::::::::::::: questions +- "How do I use the same program for multiple use-cases?" +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives +- "First objective." +:::::::::::::::::::::::::::::::::::::::::::::::: + +From the last run of our code, we can see that 500 iterations is not enough to get to a _steady state_ (a +state where the difference in temperature does not vary too much, i.e. `curdif`<`mindif`). Now, if we want to +change the number of iterations we would need to modify `niter` in the code, and compile it again. What if we +want to change the number of rows and columns in our grid to have more precision, or if we want to see the +evolution of the temperature at a different point (x,y)? The answer would be the same, modify the code and +compile it again! + +No need to say that this would be very tedious and inefficient. A better scenario would be if we can pass the +desired configuration values to our binary when it is called at the command line. The Chapel mechanism for +this is to use **_config_** variables. When a variable is declared with the `config` keyword, in addition to +`var` or `const`, like this: + +```chpl +config const niter = 500; //number of iterations +``` + +```bash +>> chpl base_solution.chpl -o base_solution +``` + +it can be initialised with a specific value, when executing the code at the command line, using the syntax: + +```bash +>> ./base_solution --niter=3000 +``` + +```output +The simulation will consider a matrix of 100 by 100 elements, +it will run up to 3000 iterations, or until the largest difference +in temperature between iterations is less than 0.0001. +You are interested in the evolution of the temperature at the +position (1,100) of the matrix... + +and here we go... +Temperature at iteration 0: 25.0 +Temperature at iteration 20: 2.0859 +Temperature at iteration 40: 1.42663 +... +Temperature at iteration 2980: 0.793969 +Temperature at iteration 3000: 0.793947 + +Final temperature at the desired position after 3000 iterations is: 0.793947 +The greatest difference in temperatures between the last two iterations was: 0.000350086 +``` + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 4: Can you do it? + +Make `n`, `x`, `y`, `mindif`, `rows` and `cols` configurable variables, and test the code simulating different +configurations. What can you conclude about the performance of the code? + +:::::::::::::::::::::::: solution + +For example, lets use a 650 x 650 grid and observe the evolution of the temperature at the position (200,300) +for 10000 iterations or until the difference of temperature between iterations is less than 0.002; also, let's +print the temperature every 1000 iterations. + +```bash +>> ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +``` + +```output +The simulation will consider a matrix of 650 by 650 elements, it will run up to 10000 iterations, or until +the largest difference in temperature between iterations is less than 0.002. You are interested in the +evolution of the temperature at the position (200,300) of the matrix... + +and here we go... +Temperature at iteration 0: 25.0 +Temperature at iteration 1000: 25.0 +Temperature at iteration 2000: 25.0 +Temperature at iteration 3000: 25.0 +Temperature at iteration 4000: 24.9998 +Temperature at iteration 5000: 24.9984 +Temperature at iteration 6000: 24.9935 +Temperature at iteration 7000: 24.9819 + +Final temperature at the desired position after 7750 iterations is: 24.9671 +The greatest difference in temperatures between the last two iterations was: 0.00199985 +``` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: keypoints +- "Config variables accept values from the command line at runtime, without you having to recompile the code." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/08-timing.md b/episodes/08-timing.md similarity index 68% rename from _episodes/08-timing.md rename to episodes/08-timing.md index 7725dd2..1a37756 100644 --- a/_episodes/08-timing.md +++ b/episodes/08-timing.md @@ -2,29 +2,28 @@ title: "Measuring code performance" teaching: 60 exercises: 30 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "How do I know how fast my code is?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "First objective." -keypoints: -- "Use UNIX `time` command or instrument your Chapel code to measure performance." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: -The code generated after Exercise 4 is the basic implementation of our -simulation. We will use it as a benchmark, to see how much we can improve the -performance when introducing the parallel programming features of the language -in the following lessons. +The code generated after Exercise 4 is the basic implementation of our simulation. We will use it as a +benchmark, to see how much we can improve the performance when introducing the parallel programming features +of the language in the following lessons. -But first, we need a quantitative way to measure the performance of our code. -The easiest way to do it is to see how long it takes to finish a simulation. -The UNIX command `time` could be used to this effect +But first, we need a quantitative way to measure the performance of our code. The easiest way to do it is to +see how long it takes to finish a simulation. The UNIX command `time` could be used to this effect -~~~ +```bash >> time ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 -~~~ -{: .bash} +``` -~~~ +```output The simulation will consider a matrix of 650 by 650 elements, it will run up to 10000 iterations, or until the largest difference in temperature between iterations is less than 0.002. @@ -44,28 +43,24 @@ Temperature at iteration 7000: 24.9819 Final temperature at the desired position after 7750 iterations is: 24.9671 The greatest difference in temperatures between the last two iterations was: 0.00199985 - real 0m20.381s user 0m20.328s sys 0m0.053s -~~~ -{: .output} +``` -The real time is what interests us. Our code is taking around 20 seconds from -the moment it is called at the command line until it returns. +The real time is what interests us. Our code is taking around 20 seconds from the moment it is called at the +command line until it returns. -Some times, however, it could be useful to take the execution time of specific -parts of the code. This can be achieved by modifying the code to output the -information that we need. This process is called **_instrumentation of code_**. +Some times, however, it could be useful to take the execution time of specific parts of the code. This can be +achieved by modifying the code to output the information that we need. This process is called +**_instrumentation of code_**. -An easy way to instrument our code with Chapel is by using the module `Time`. -**_Modules_** in Chapel are libraries of useful functions and methods that can -be used once the module is loaded. To load a module we use the keyword `use` -followed by the name of the module. Once the Time module is loaded we can -create a variable of the type `Timer`, and use the methods `start`,`stop`and -`elapsed` to instrument our code. +An easy way to instrument our code with Chapel is by using the module `Time`. **_Modules_** in Chapel are +libraries of useful functions and methods that can be used once the module is loaded. To load a module we use +the keyword `use` followed by the name of the module. Once the Time module is loaded we can create a variable +of the type `Timer`, and use the methods `start`,`stop`and `elapsed` to instrument our code. -~~~ +```chpl use Time; var watch: Timer; watch.start(); @@ -83,16 +78,14 @@ watch.stop(); writeln('\nThe simulation took ',watch.elapsed(),' seconds'); writeln('Final temperature at the desired position after ',c,' iterations is: ',temp[x,y]); writeln('The greatest difference in temperatures between the last two iterations was: ',curdif,'\n'); -~~~ -{: .source} +``` -~~~ +```bash >> chpl base_solution.chpl -o base_solution >> ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 -~~~ -{: .bash} +``` -~~~ +```output The simulation will consider a matrix of 650 by 650 elements, it will run up to 10000 iterations, or until the largest difference in temperature between iterations is less than 0.002. @@ -112,5 +105,8 @@ Temperature at iteration 7000: 24.9819 The simulation took 20.1621 seconds Final temperature at the desired position after 7750 iterations is: 24.9671 The greatest difference in temperatures between the last two iterations was: 0.00199985 -~~~ -{: .output} +``` + +::::::::::::::::::::::::::::::::::::: keypoints +- "Use UNIX `time` command or instrument your Chapel code to measure performance." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/episodes/11-parallel-intro.md b/episodes/11-parallel-intro.md new file mode 100644 index 0000000..f22bb80 --- /dev/null +++ b/episodes/11-parallel-intro.md @@ -0,0 +1,101 @@ +--- +title: "Intro to parallel computing" +teaching: 60 +exercises: 30 +--- + +:::::::::::::::::::::::::::::::::::::: questions +- "How does parallel processing work?" +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives +- "First objective." +:::::::::::::::::::::::::::::::::::::::::::::::: + +The basic concept of parallel computing is simple to understand: we divide our job into tasks that can be +executed at the same time, so that we finish the job in a fraction of the time that it would have taken if the +tasks were executed one by one. Implementing parallel computations, however, is not always easy, nor +possible... + +Consider the following analogy: + +Suppose that we want to paint the four walls in a room. We'll call this the *problem*. We can divide our +problem into 4 different tasks: paint each of the walls. In principle, our 4 tasks are independent from each +other in the sense that we don't need to finish one to start one another. We say that we have 4 **_concurrent +tasks_**; the tasks can be executed within the same time frame. However, this does not mean that the tasks +can be executed simultaneously or in parallel. It all depends on the amount of resources that we have for the +tasks. If there is only one painter, this guy could work for a while in one wall, then start painting another +one, then work for a little bit on the third one, and so on. **_The tasks are being executed concurrently but +not in parallel_**. If we have two painters for the job, then more parallelism can be introduced. Four +painters could executed the tasks **_truly in parallel_**. + +::::::::::::::::::::::::::::::::::::: callout + +Think of the CPU cores as the painters or workers that will execute your concurrent tasks + +:::::::::::::::::::::::::::::::::::::::::::::::: + +Now imagine that all workers have to obtain their paint from a central dispenser located at the middle of the +room. If each worker is using a different colour, then they can work **_asynchronously_**, however, if they +use the same colour, and two of them run out of paint at the same time, then they have to **_synchronise_** to +use the dispenser: One must wait while the other is being serviced. + +::::::::::::::::::::::::::::::::::::: callout + +Think of the shared memory in your computer as the central dispenser for all your workers + +:::::::::::::::::::::::::::::::::::::::::::::::: + +Finally, imagine that we have 4 paint dispensers, one for each worker. In this scenario, each worker can +complete their task totally on their own. They don't even have to be in the same room, they could be painting +walls of different rooms in the house, in different houses in the city, and different cities in the +country. We need, however, a communication system in place. Suppose that worker A, for some reason, needs a +colour that is only available in the dispenser of worker B, they must then synchronise: worker A must request +the paint of worker B and worker B must respond by sending the required colour. + +::::::::::::::::::::::::::::::::::::: callout + +Think of the memory distributed on each node of a cluster as the different dispensers for your workers + +:::::::::::::::::::::::::::::::::::::::::::::::: + +A **_fine-grained_** parallel code needs lots of communication or synchronisation between tasks, in contrast +with a **_coarse-grained_** one. An **_embarrassingly parallel_** problem is one where all tasks can be +executed completely independent from each other (no communications required). + +## Parallel programming in Chapel + +Chapel provides high-level abstractions for parallel programming no matter the grain size of your tasks, +whether they run in a shared memory or a distributed memory environment, or whether they are executed +concurrently or truly in parallel. As a programmer you can focus in the algorithm: how to divide the problem +into tasks that make sense in the context of the problem, and be sure that the high-level implementation will +run on any hardware configuration. Then you could consider the details of the specific system you are going to +use (whether it is shared or distributed, the number of cores, etc.) and tune your code/algorithm to obtain a +better performance. + +::::::::::::::::::::::::::::::::::::: callout + +To this effect, **_concurrency_** (the creation and execution of multiple tasks), and **_locality_** (in +which set of resources these tasks are executed) are orthogonal concepts in Chapel. + +:::::::::::::::::::::::::::::::::::::::::::::::: + +In summary, we can have a set of several tasks; these tasks could be running: + +1. concurrently by the same processor in a single compute node, +2. in parallel by several processors in a single compute node, +3. in parallel by several processors distributed in different compute nodes, or +4. serially (one by one) by several processors distributed in different compute nodes. + +Similarly, each of these tasks could be using variables located in: + +1. the local memory on the compute node where it is running, or +2. on distributed memory located in other compute nodes. + +And again, Chapel could take care of all the stuff required to run our algorithm in most of the scenarios, but +we can always add more specific detail to gain performance when targeting a particular scenario. + +::::::::::::::::::::::::::::::::::::: keypoints +- "Concurrency and locality are orthogonal concepts in Chapel: where the tasks are running may not be + indicative of when they run, and you can control both in Chapel." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md new file mode 100644 index 0000000..df0c58c --- /dev/null +++ b/episodes/12-fire-forget-tasks.md @@ -0,0 +1,392 @@ +--- +title: "Fire-and-forget tasks" +teaching: 60 +exercises: 30 +--- + +:::::::::::::::::::::::::::::::::::::: questions +- "How do execute work in parallel?" +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives +- "First objective." +:::::::::::::::::::::::::::::::::::::::::::::::: + +A Chapel program always start as a single main thread. You can then start concurrent tasks with the `begin` +statement. A task spawned by the `begin` statement will run in a different thread while the main thread +continues its normal execution. Consider the following example: + +```chpl +var x=0; + +writeln("This is the main thread starting first task"); +begin +{ + var c=0; + while c<100 + { + c+=1; + writeln('thread 1: ',x+c); + } +} + +writeln("This is the main thread starting second task"); +begin +{ + var c=0; + while c<100 + { + c+=1; + writeln('thread 2: ',x+c); + } +} + +writeln('this is main thread, I am done...'); +``` + +```bash +>> chpl begin_example.chpl -o begin_example +>> ./begin_example +``` + +```output +This is the main thread starting first task +This is the main thread starting second task +this is main thread, I am done... +thread 2: 1 +thread 2: 2 +thread 2: 3 +thread 2: 4 +thread 2: 5 +thread 2: 6 +thread 2: 7 +thread 2: 8 +thread 2: 9 +thread 2: 10 +thread 2: 11 +thread 2: 12 +thread 2: 13 +thread 1: 1 +thread 2: 14 +thread 1: 2 +thread 2: 15 +... +thread 2: 99 +thread 1: 97 +thread 2: 100 +thread 1: 98 +thread 1: 99 +thread 1: 100 +``` + +As you can see the order of the output is not what we would expected, and actually it is completely +unpredictable. This is a well known effect of concurrent tasks accessing the same shared resource at the same +time (in this case the screen); the system decides in which order the tasks could write to the screen. + +::::::::::::::::::::::::::::::::::::::: discussion + +## Discussion + +- What would happen if in the last code we declare `c` in the main thread? +- What would happen if we try to modify the value of `x` inside a begin statement? + +Discuss your observations. + +::::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: callout + +All variables have a **_scope_** in which they can be used. The variables declared inside a concurrent tasks +are accessible only by the task. The variables declared in the main task can be read everywhere, but Chapel +won't allow two concurrent tasks to try to modify them. + +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::: discussion + +## Try this ... + +Are the concurrent tasks, spawned by the last code, running truly in parallel? + +The answer is: it depends on the number of cores available to your job. To verify this, let's modify the code +to get the tasks into an infinite loop. + +```chpl +begin +{ + var c=0; + while c>-1 + { + c+=1; + //writeln('thread 1: ',x+c); + } +} +``` + +Now submit your job asking for different amount of resources, and use system tools such as `top`or `ps` to +monitor the execution of the code. + +::::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: callout + +To maximise performance, start as many tasks as cores are available. + +:::::::::::::::::::::::::::::::::::::::::::::::: + +A slightly more structured way to start concurrent tasks in Chapel is by using the `cobegin`statement. Here +you can start a block of concurrent tasks, one for each statement inside the curly brackets. The main +difference between the `begin`and `cobegin` statements is that with the `cobegin`, all the spawned tasks are +synchronised at the end of the statement, i.e. the main thread won't continue its execution until all tasks +are done. + +```chpl +var x=0; +writeln("This is the main thread, my value of x is ",x); + +cobegin +{ + { + var x=5; + writeln("this is task 1, my value of x is ",x); + } + writeln("this is task 2, my value of x is ",x); +} + +writeln("this message won't appear until all tasks are done..."); +``` + +```bash +>> chpl cobegin_example.chpl -o cobegin_example +>> ./cobegin_example +``` + +```output +This is the main thread, my value of x is 0 +this is task 2, my value of x is 0 +this is task 1, my value of x is 5 +this message won't appear until all tasks are done... +``` + +As you may have conclude from the Discussion exercise above, the variables declared inside a task are +accessible only by the task, while those variables declared in the main task are accessible to all tasks. + +The last, and most useful way to start concurrent/parallel tasks in Chapel, is the `coforall` loop. This is a +combination of the for-loop and the `cobegin`statements. The general syntax is: + +```chpl +coforall index in iterand +{instructions} +``` + +This will start a new task, for each iteration. Each tasks will then perform all the instructions inside the +curly brackets. Each task will have a copy of the variable **_index_** with the corresponding value yielded by +the iterand. This index allows us to _customise_ the set of instructions for each particular task. + +```chpl +var x=1; +config var numoftasks=2; + +writeln("This is the main task: x = ",x); + +coforall taskid in 1..numoftasks do +{ + var c=taskid+1; + writeln("this is task ",taskid,": x + ",taskid," = ",x+taskid,". My value of c is: ",c); +} + +writeln("this message won't appear until all tasks are done..."); +``` + +```bash +> > chpl coforall_example.chpl -o coforall_example +> > ./coforall_example --numoftasks=5 +``` + +```output +This is the main task: x = 1 +this is task 5: x + 5 = 6. My value of c is: 6 +this is task 2: x + 2 = 3. My value of c is: 3 +this is task 4: x + 4 = 5. My value of c is: 5 +this is task 3: x + 3 = 4. My value of c is: 4 +this is task 1: x + 1 = 2. My value of c is: 2 +this message won't appear until all tasks are done... +``` + +Notice how we are able to customise the instructions inside the coforall, to give different results depending +on the task that is executing them. Also, notice how, once again, the variables declared outside the coforall +can be read by all tasks, while the variables declared inside, are available only to the particular task. + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 1: Can you do it? + +Would it be possible to print all the messages in the right order? Modify the code in the last example as +required. + +Hint: you can use an array of strings declared in the main task, where all the concurrent tasks could write +their messages in the corresponding position. Then, at the end, have the main task printing all elements of +the array in order. + +:::::::::::::::::::::::: solution + +The following code is a possible solution: + +```chpl +var x=1; +config var numoftasks=2; +var messages: [1..numoftasks] string; + +writeln("This is the main task: x = ",x); + +coforall taskid in 1..numoftasks do +{ + var c=taskid+1; + var s="this is task "+taskid+": x + "+taskid+" = "+(x+taskid)+". My value of c is: "+c; + messages[taskid]=s; +} + +for i in 1..numoftasks do writeln(messages[i]); +writeln("this message won't appear until all tasks are done..."); +``` + +```bash +chpl exercise_coforall.chpl -o exercise_coforall +./exercise_coforall --numoftasks=5 +``` + +```output +This is the main task: x = 1 +this is task 1: x + 1 = 2. My value of c is: 2 +this is task 2: x + 2 = 3. My value of c is: 3 +this is task 3: x + 3 = 4. My value of c is: 4 +this is task 4: x + 4 = 5. My value of c is: 5 +this is task 5: x + 5 = 6. My value of c is: 6 +this message won't appear until all tasks are done... +``` + +Note that `+` is a **_polymorphic_** operand in Chapel. In this case it concatenates `strings` with `integers` +(which are transformed to strings). + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 2: Can you do it? + +Consider the following code: + +```chpl +use Random; +config const nelem=5000000000; +var x: [1..n] int; +fillRandom(x); //fill array with random numbers +var mymax=0; + +// here put your code to find mymax + +writeln("the maximum value in x is: ",mymax); +``` + +Write a parallel code to find the maximum value in the array x. + +:::::::::::::::::::::::: solution + +```chpl +config const numoftasks=12; +const n=nelem/numoftasks; +const r=nelem-n*numoftasks; + +var d: [0..numoftasks-1] real; + +coforall taskid in 0..numoftasks-1 do +{ + var i: int; + var f: int; + if taskidd[taskid] then d[taskid]=x[c]; + } +} +for i in 0..numoftasks-1 do +{ + if d[i]>mymax then mymax=d[i]; +} +``` + +```bash +> > chpl --fast exercise_coforall_2.chpl -o exercise_coforall_2 +> > ./exercise_coforall_2 +``` + +```output +the maximum value in x is: 1.0 +``` + +We use the coforall to spawn tasks that work concurrently in a fraction of the array. The trick here is to +determine, based on the _taskid_, the initial and final indices that the task will use. Each task obtains the +maximum in its fraction of the array, and finally, after the coforall is done, the main task obtains the +maximum of the array from the maximums of all tasks. + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::: discussion + +## Discussion + +Run the code of last Exercise using different number of tasks, and different sizes of the array _x_ to see how +the execution time changes. For example: + +```bash +time ./exercise_coforall_2 --nelem=3000 --numoftasks=4 +``` + +Discuss your observations. Is there a limit on how fast the code could run? + +::::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::: discussion + +## Try this ... + +Substitute the code to find _mymax_ in the last exercise with: + +```chpl +mymax=max reduce x; +``` + +Time the execution of the original code and this new one. How do they compare? + +::::::::::::::::::::::::::::::::::::::::::::::::::: + + +::::::::::::::::::::::::::::::::::::: callout + +It is always a good idea to check whether there is _built-in_ functions or methods in the used language, that +can do what we want to do as efficiently (or better) than our house-made code. In this case, the _reduce_ +statement reduces the given array to a single number using the given operation (in this case max), and it is +parallelized and optimised to have a very good performance. + +:::::::::::::::::::::::::::::::::::::::::::::::: + + +The code in these last Exercises somehow _synchronise_ the tasks to obtain the desired result. In addition, +Chapel has specific mechanisms task synchronisation, that could help us to achieve fine-grained +parallelization. + +::::::::::::::::::::::::::::::::::::: keypoints +- "Use `begin` or `cobegin` or `coforall` to spawn new tasks." +- "You can run more than one task per core, as the number of cores on a node is limited." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/13-synchronization.md b/episodes/13-synchronization.md similarity index 61% rename from _episodes/13-synchronization.md rename to episodes/13-synchronization.md index 60ca80a..161cf33 100644 --- a/_episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -2,22 +2,21 @@ title: "Synchronising tasks" teaching: 60 exercises: 30 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "How should I access my data in parallel?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "First objective." -keypoints: -- "You can explicitly synchronise tasks with `sync` statement." -- "You can also use sync and atomic variables to synchronise tasks." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: -The keyword `sync` provides all sorts of mechanisms to synchronise tasks in -Chapel. +The keyword `sync` provides all sorts of mechanisms to synchronise tasks in Chapel. -We can simply use `sync` to force the _parent_ task to stop and wait until its -_spawned-child-task_ ends. +We can simply use `sync` to force the _parent_ task to stop and wait until its _spawned-child-task_ ends. -~~~ +```chpl var x=0; writeln("This is the main thread starting a synchronous task"); @@ -47,16 +46,14 @@ begin } writeln('this is main thread, I am done...'); -~~~ -{: .source} +``` -~~~ +```bash >> chpl sync_example_1.chpl -o sync_example_1 >> ./sync_example_1 -~~~ -{: .bash} +``` -~~~ +```output This is the main thread starting a synchronous task thread 1: 1 thread 1: 2 @@ -81,67 +78,68 @@ thread 2: 7 thread 2: 8 thread 2: 9 thread 2: 10 -~~~ -{: .output} +``` -> ## Discussion -> -> What would happen if we write instead -> ~~~ -> begin -> { -> sync -> { -> var c=0; -> while c<10 -> { -> c+=1; -> writeln('thread 1: ',x+c); -> } -> } -> } -> writeln("The first task is done..."); -> ~~~ -> {: .source} -{: .discussion} - -> ## Exercise 3 -> -> Use `begin` and `sync` statements to reproduce the functionality of `cobegin` -> in `cobegin_example.chpl`. -> -> > ## Solution -> > ~~~ -> > var x=0; -> > writeln("This is the main thread, my value of x is ",x); -> > -> > sync -> > { -> > begin -> > { -> > var x=5; -> > writeln("this is task 1, my value of x is ",x); -> > } -> > begin writeln("this is task 2, my value of x is ",x); -> > } -> > -> > writeln("this message won't appear until all tasks are done..."); -> > ~~~ -> > {: .source} -> {: .solution} -{: .challenge} - -A more elaborated and powerful use of `sync` is as a type qualifier for -variables. When a variable is declared as _sync_, a state that can be -**_full_** or **_empty_** is associated to it. - -To assign a new value to a _sync_ variable, its state must be _empty_ (after -the assignment operation is completed, the state will be set as _full_). On the -contrary, to read a value from a _sync_ variable, its state must be _full_ -(after the read operation is completed, the state will be set as _empty_ -again). - -~~~ +::::::::::::::::::::::::::::::::::::::: discussion + +## Discussion + +What would happen if we write instead + +```chpl +begin +{ + sync + { + var c=0; + while c<10 + { + c+=1; + writeln('thread 1: ',x+c); + } + } +} +writeln("The first task is done..."); +``` + +::::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 3: Can you do it? + +Use `begin` and `sync` statements to reproduce the functionality of `cobegin` in `cobegin_example.chpl`. + +:::::::::::::::::::::::: solution + +```chpl +var x=0; +writeln("This is the main thread, my value of x is ",x); + +sync +{ + begin + { + var x=5; + writeln("this is task 1, my value of x is ",x); + } + begin writeln("this is task 2, my value of x is ",x); + } + +writeln("this message won't appear until all tasks are done..."); +``` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +A more elaborated and powerful use of `sync` is as a type qualifier for variables. When a variable is declared +as _sync_, a state that can be **_full_** or **_empty_** is associated to it. + +To assign a new value to a _sync_ variable, its state must be _empty_ (after the assignment operation is +completed, the state will be set as _full_). On the contrary, to read a value from a _sync_ variable, its +state must be _full_ (after the read operation is completed, the state will be set as _empty_ again). + +```chpl var x: sync int, a: int; writeln("this is main task launching a new task"); begin { @@ -153,16 +151,14 @@ begin { writeln("this is main task after launching new task... I will wait until it is done"); a = x; // don't run this line until the variable x is written in the other task writeln("and now it is done"); -~~~ -{: .source} +``` -~~~ +```bash >> chpl sync_example_2.chpl -o sync_example_2 >> ./sync_example_2 -~~~ -{: .bash} +``` -~~~ +```output this is main task launching a new task this is main task after launching new task... I will wait until it is done this is new task working: 1 @@ -177,22 +173,23 @@ this is new task working: 9 this is new task working: 10 New task finished and now it is done -~~~ -{: .output} +``` -> ## Discussion -> -> What would happen if we assign a value to _x_ right before launching the new -> task? What would happen if we assign a value to _x_ right before launching -> the new task and after the _writeln("and now it is done");_ statement? -> -> Discuss your observations. -{: .discussion} +::::::::::::::::::::::::::::::::::::::: discussion + +## Discussion + +What would happen if we assign a value to _x_ right before launching the new task? What would happen if we +assign a value to _x_ right before launching the new task and after the _writeln("and now it is done");_ +statement? -There are a number of methods defined for _sync_ variables. Suppose _x_ is a -sync variable of a given type, +Discuss your observations. -~~~ +::::::::::::::::::::::::::::::::::::::::::::::::::: + +There are a number of methods defined for _sync_ variables. Suppose _x_ is a sync variable of a given type, + +```chpl // general methods x.reset() //will set the state as empty and the value as the default of x's type x.isfull() //will return true is the state of x is full, false if it is empty @@ -210,19 +207,16 @@ x.readFF() //will block until the state of x is full, //non-blocking read and write methods x.writeXF(value) //will assign the value no matter the state of x, and then set the state as full x.readXX() //will return the value of x regardless its state. The state will remain unchanged -~~~ -{: .source} - -Chapel also implements **_atomic_** operations with variables declared as -`atomic`, and this provides another option to synchronise tasks. Atomic -operations run completely independently of any other thread or process. This -means that when several tasks try to write an atomic variable, only one will -succeed at a given moment, providing implicit synchronisation between them. -There is a number of methods defined for atomic variables, among them `sub()`, -`add()`, `write()`, `read()`, and `waitfor()` are very useful to establish +``` + +Chapel also implements **_atomic_** operations with variables declared as `atomic`, and this provides another +option to synchronise tasks. Atomic operations run completely independently of any other thread or +process. This means that when several tasks try to write an atomic variable, only one will succeed at a given +moment, providing implicit synchronisation between them. There is a number of methods defined for atomic +variables, among them `sub()`, `add()`, `write()`, `read()`, and `waitfor()` are very useful to establish explicit synchronisation between tasks, as showed in the next code: -~~~ +```chpl var lock: atomic int; const numtasks=5; @@ -235,16 +229,14 @@ coforall id in 1..numtasks lock.waitFor(numtasks); //then it waits for lock to be equal numtasks (which will happen when all tasks say hello) writeln("task ",id," is done..."); } -~~~ -{: .source} +``` -~~~ +```bash >> chpl atomic_example.chpl -o atomic_example >> ./atomic_example -~~~ -{: .bash} +``` -~~~ +```output greetings form task 4... I am waiting for all tasks to say hello greetings form task 5... I am waiting for all tasks to say hello greetings form task 2... I am waiting for all tasks to say hello @@ -255,14 +247,17 @@ task 5 is done... task 2 is done... task 3 is done... task 4 is done... -~~~ -{: .output} +``` > ## Try this... > -> Comment out the line `lock.waitfor(numtasks)` in the code above to clearly -> observe the effect of the task synchronisation. -{: .challenge} +> Comment out the line `lock.waitfor(numtasks)` in the code above to clearly observe the effect of the task +> synchronisation. -Finally, with all the material studied so far, we should be ready to -parallelize our code for the simulation of the heat transfer equation. +Finally, with all the material studied so far, we should be ready to parallelize our code for the simulation +of the heat transfer equation. + +::::::::::::::::::::::::::::::::::::: keypoints +- "You can explicitly synchronise tasks with `sync` statement." +- "You can also use sync and atomic variables to synchronise tasks." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/14-parallel-case-study.md b/episodes/14-parallel-case-study.md similarity index 62% rename from _episodes/14-parallel-case-study.md rename to episodes/14-parallel-case-study.md index ee0672f..af730aa 100644 --- a/_episodes/14-parallel-case-study.md +++ b/episodes/14-parallel-case-study.md @@ -2,26 +2,26 @@ title: "Task parallelism with Chapel" teaching: 60 exercises: 30 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "How do I write parallel code for a real use case?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "First objective." -keypoints: -- "There are many ways to implement task parallelism for the diffusion solver." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: -The parallelization of our base solution for the heat transfer equation can be -achieved following the ideas of Exercise 2. The entire grid of points can be -divided and assigned to multiple tasks. Each tasks should compute the new -temperature of its assigned points, and then we must perform a **_reduction_**, -over the whole grid, to update the greatest difference in temperature. +The parallelization of our base solution for the heat transfer equation can be achieved following the ideas of +Exercise 2. The entire grid of points can be divided and assigned to multiple tasks. Each tasks should compute +the new temperature of its assigned points, and then we must perform a **_reduction_**, over the whole grid, +to update the greatest difference in temperature. -For the reduction of the grid we can simply use the `max reduce` statement, -which is already parallelized. Now, let's divide the grid into `rowtasks` x -`coltasks` sub-grids, and assign each sub-grid to a task using the `coforall` -loop (we will have `rowtasks*coltasks` tasks in total). +For the reduction of the grid we can simply use the `max reduce` statement, which is already +parallelized. Now, let's divide the grid into `rowtasks` x `coltasks` sub-grids, and assign each sub-grid to a +task using the `coforall` loop (we will have `rowtasks*coltasks` tasks in total). -~~~ +```chpl config const rowtasks = 2; config const coltasks = 2; @@ -43,15 +43,13 @@ while (c=mindif) do { if c%n == 0 then writeln('Temperature at iteration ',c,': ',temp[x,y]); } -~~~ -{: .source} +``` -Note that now the nested for loops run from `rowi` to `rowf` and from `coli` to -`colf` which are, respectively, the initial and final row and column of the -sub-grid associated to the task `taskid`. To compute these limits, based on -`taskid`, we can again follow the same ideas as in Exercise 2. +Note that now the nested for loops run from `rowi` to `rowf` and from `coli` to `colf` which are, +respectively, the initial and final row and column of the sub-grid associated to the task `taskid`. To compute +these limits, based on `taskid`, we can again follow the same ideas as in Exercise 2. -~~~ +```chpl config const rowtasks = 2; config const coltasks = 2; @@ -94,23 +92,19 @@ while (c=mindif) do { for j in coli..colf do { ... } -~~~ -{: .source} +``` -As you can see, to divide a data set (the array `temp` in this case) between -concurrent tasks, could be cumbersome. Chapel provides high-level abstractions -for data parallelism that take care of all the data distribution for us. We -will study data parallelism in the following lessons, but for now, let's -compare the benchmark solution with our `coforall` parallelization to see how -the performance improved. +As you can see, to divide a data set (the array `temp` in this case) between concurrent tasks, could be +cumbersome. Chapel provides high-level abstractions for data parallelism that take care of all the data +distribution for us. We will study data parallelism in the following lessons, but for now, let's compare the +benchmark solution with our `coforall` parallelization to see how the performance improved. -~~~ +```bash > > chpl --fast parallel_solution_1.chpl -o parallel1 > > ./parallel1 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 -~~~ -{: .bash} +``` -~~~ +```output The simulation will consider a matrix of 650 by 650 elements, it will run up to 10000 iterations, or until the largest difference in temperature between iterations is less than 0.002. @@ -129,27 +123,21 @@ Temperature at iteration 7000: 24.9819 The simulation took 17.0193 seconds Final temperature at the desired position after 7750 iterations is: 24.9671 The greatest difference in temperatures between the last two iterations was: 0.00199985 -~~~ -{: .output} - -This parallel solution, using 4 parallel tasks, took around 17 seconds to -finish. Compared with the ~20 seconds needed by the benchmark solution, seems -not very impressive. To understand the reason, let's analyse the code's flow. -When the program starts, the main thread does all the declarations and -initialisations, and then, it enters the main loop of the simulation (the -**_while loop_**). Inside this loop, the parallel tasks are launched for the -first time. When these tasks finish their computations, the main task resumes -its execution, it updates `curdif`, and everything is repeated again. So, in -essence, parallel tasks are launched and resumed 7750 times, which introduces a -significant amount of overhead (the time the system needs to effectively start -and destroy threads in the specific hardware, at each iteration of the while -loop). - -Clearly, a better approach would be to launch the parallel tasks just once, and -have them executing all the simulations, before resuming the main task to print -the final results. - -~~~ +``` + +This parallel solution, using 4 parallel tasks, took around 17 seconds to finish. Compared with the ~20 +seconds needed by the benchmark solution, seems not very impressive. To understand the reason, let's analyse +the code's flow. When the program starts, the main thread does all the declarations and initialisations, and +then, it enters the main loop of the simulation (the **_while loop_**). Inside this loop, the parallel tasks +are launched for the first time. When these tasks finish their computations, the main task resumes its +execution, it updates `curdif`, and everything is repeated again. So, in essence, parallel tasks are launched +and resumed 7750 times, which introduces a significant amount of overhead (the time the system needs to +effectively start and destroy threads in the specific hardware, at each iteration of the while loop). + +Clearly, a better approach would be to launch the parallel tasks just once, and have them executing all the +simulations, before resuming the main task to print the final results. + +```chpl config const rowtasks = 2; config const coltasks = 2; @@ -200,28 +188,24 @@ coforall taskid in 0..coltasks*rowtasks-1 do { //print temperature in desired position } } -~~~ -{: .source} +``` -The problem with this approach is that now we have to explicitly synchronise -the tasks. Before, `curdif` and `past_temp` were updated only by the main task -at each iteration; similarly, only the main task was printing results. Now, all -these operations must be carried inside the coforall loop, which imposes the -need of synchronisation between tasks. +The problem with this approach is that now we have to explicitly synchronise the tasks. Before, `curdif` and +`past_temp` were updated only by the main task at each iteration; similarly, only the main task was printing +results. Now, all these operations must be carried inside the coforall loop, which imposes the need of +synchronisation between tasks. The synchronisation must happen at two points: -1. We need to be sure that all tasks have finished with the computations of - their part of the grid `temp`, before updating `curdif` and `past_temp` - safely. -2. We need to be sure that all tasks use the updated value of `curdif` to - evaluate the condition of the while loop for the next iteration. +1. We need to be sure that all tasks have finished with the computations of their part of the grid `temp`, + before updating `curdif` and `past_temp` safely. +2. We need to be sure that all tasks use the updated value of `curdif` to evaluate the condition of the while + loop for the next iteration. -To update `curdif` we could have each task computing the greatest difference in -temperature in its associated sub-grid, and then, after the synchronisation, -have only one task reducing all the sub-grids' maximums. +To update `curdif` we could have each task computing the greatest difference in temperature in its associated +sub-grid, and then, after the synchronisation, have only one task reducing all the sub-grids' maximums. -~~~ +```chpl var curdif: atomic real; var myd: [0..coltasks*rowtasks-1] real; ... @@ -255,61 +239,58 @@ coforall taskid in 0..coltasks*rowtasks-1 do // here comes the synchronisation of tasks again } } -~~~ -{: .source} - -> ## Exercise 4 -> -> Use `sync` or `atomic` variables to implement the synchronisation required in -> the code above. -> -> > ## Solution -> > -> > One possible solution is to use an atomic variable as a _lock_ that opens -> > (using the `waitFor` method) when all the tasks complete the required -> > instructions -> > -> > ~~~ -> > var lock: atomic int; -> > lock.write(0); -> > ... -> > //this is the main loop of the simulation -> > curdif.write(mindif); -> > coforall taskid in 0..coltasks*rowtasks-1 do -> > { -> > ... -> > while (c=mindif) do -> > { -> > ... -> > myd[taskid]=myd2 -> > -> > //here comes the synchronisation of tasks -> > lock.add(1); -> > lock.waitFor(coltasks*rowtasks); -> > -> > past_temp[rowi..rowf,coli..colf]=temp[rowi..rowf,coli..colf]; -> > ... -> > -> > //here comes the synchronisation of tasks again -> > lock.sub(1); -> > lock.waitFor(0); -> > } -> > } -> > ~~~ -> > {: .source} -> {: .solution} -{: .challenge} - -Using the solution in the Exercise 4, we can now compare the performance with -the benchmark solution - -~~~ +``` + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 4: Can you do it? + +Use `sync` or `atomic` variables to implement the synchronisation required in the code above. + +:::::::::::::::::::::::: solution + +One possible solution is to use an atomic variable as a _lock_ that opens (using the `waitFor` method) when +all the tasks complete the required instructions + +```chpl +var lock: atomic int; +lock.write(0); +... +//this is the main loop of the simulation +curdif.write(mindif); +coforall taskid in 0..coltasks*rowtasks-1 do +{ + ... + while (c=mindif) do + { + ... + myd[taskid]=myd2 + + //here comes the synchronisation of tasks + lock.add(1); + lock.waitFor(coltasks*rowtasks); + + past_temp[rowi..rowf,coli..colf]=temp[rowi..rowf,coli..colf]; + ... + + //here comes the synchronisation of tasks again + lock.sub(1); + lock.waitFor(0); + } +} +``` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +Using the solution in the Exercise 4, we can now compare the performance with the benchmark solution + +```bash >> chpl --fast parallel_solution_2.chpl -o parallel2 >> ./parallel2 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 -~~~ -{: .bash} +``` -~~~ +```output The simulation will consider a matrix of 650 by 650 elements, it will run up to 10000 iterations, or until the largest difference in temperature between iterations is less than 0.002. @@ -328,16 +309,14 @@ Temperature at iteration 7000: 24.9819 The simulation took 4.2733 seconds Final temperature at the desired position after 7750 iterations is: 24.9671 The greatest difference in temperatures between the last two iterations was: 0.00199985 -~~~ -{: .output} +``` to see that we now have a code that performs 5x faster. -We finish this section by providing another, elegant version of the 2D heat -transfer solver (without time stepping) using data parallelism on a single -locale: +We finish this section by providing another, elegant version of the 2D heat transfer solver (without time +stepping) using data parallelism on a single locale: -~~~ +```chpl const n = 100, stride = 20; var T: [0..n+1, 0..n+1] real; var Tnew: [1..n,1..n] real; @@ -354,7 +333,10 @@ coforall (i,j) in {1..n,1..n} by (stride,stride) { // 5x5 decomposition into 20x } } } -~~~ -{: .source} +``` We will study data parallelism in more detail in the next section. + +::::::::::::::::::::::::::::::::::::: keypoints +- "There are many ways to implement task parallelism for the diffusion solver." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/21-locales.md b/episodes/21-locales.md similarity index 51% rename from _episodes/21-locales.md rename to episodes/21-locales.md index 3371f03..0150154 100644 --- a/_episodes/21-locales.md +++ b/episodes/21-locales.md @@ -2,165 +2,143 @@ title: "Running code on multiple machines" teaching: 120 exercises: 60 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "What is a locale?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "First objective." -keypoints: -- "Locale in Chapel is a shared-memory node on a cluster." -- "We can cycle in serial or parallel through all locales." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: -So far we have been working with single-locale Chapel codes that may run on one -or many cores on a single compute node, making use of the shared memory space -and accelerating computations by launching concurrent tasks on individual cores -in parallel. Chapel codes can also run on multiple nodes on a compute cluster. -In Chapel this is referred to as *multi-locale* execution. +So far we have been working with single-locale Chapel codes that may run on one or many cores on a single +compute node, making use of the shared memory space and accelerating computations by launching concurrent +tasks on individual cores in parallel. Chapel codes can also run on multiple nodes on a compute cluster. In +Chapel this is referred to as *multi-locale* execution. -If you work inside a Chapel Docker container, e.g., chapel/chapel-gasnet, the -container environment simulates a multi-locale cluster, so you would compile -and launch multi-locale Chapel codes directly by specifying the number of -locales with `-nl` flag: +If you work inside a Chapel Docker container, e.g., chapel/chapel-gasnet, the container environment simulates +a multi-locale cluster, so you would compile and launch multi-locale Chapel codes directly by specifying the +number of locales with `-nl` flag: -~~~ +```bash $ chpl --fast mycode.chpl -o mybinary $ ./mybinary -nl 4 -~~~ -{:.bash} - -Inside the Docker container on multiple locales your code will not run any -faster than on a single locale, since you are emulating a virtual cluster, and -all tasks run on the same physical node. To achieve actual speedup, you need to -run your parallel multi-locale Chapel code on a real physical cluster which we -hope you have access to for this session. - -On a real HPC cluster you would need to submit either an interactive or a batch -job asking for several nodes and then run a multi-locale Chapel code inside -that job. In practice, the exact commands depend on how the multi-locale Chapel -was built on the cluster. - -When you compile a Chapel code with the multi-locale Chapel compiler, two -binaries will be produced. One is called `mybinary` and is a launcher binary -used to submit the real executable `mybinary_real`. If the Chapel environment -is configured properly with the launcher for the cluster's physical -interconnect (which might not be always possible due to a number of factors), -then you would simply compile the code and use the launcher binary `mybinary` -to submit the job to the queue: - -~~~ +``` + +Inside the Docker container on multiple locales your code will not run any faster than on a single locale, +since you are emulating a virtual cluster, and all tasks run on the same physical node. To achieve actual +speedup, you need to run your parallel multi-locale Chapel code on a real physical cluster which we hope you +have access to for this session. + +On a real HPC cluster you would need to submit either an interactive or a batch job asking for several nodes +and then run a multi-locale Chapel code inside that job. In practice, the exact commands depend on how the +multi-locale Chapel was built on the cluster. + +When you compile a Chapel code with the multi-locale Chapel compiler, two binaries will be produced. One is +called `mybinary` and is a launcher binary used to submit the real executable `mybinary_real`. If the Chapel +environment is configured properly with the launcher for the cluster's physical interconnect (which might not +be always possible due to a number of factors), then you would simply compile the code and use the launcher +binary `mybinary` to submit the job to the queue: + +```bash $ chpl --fast mycode.chpl -o mybinary $ ./mybinary -nl 2 -~~~ -{: .bash} - -The exact parameters of the job such as the maximum runtime and the requested -memory can be specified with Chapel environment variables. One possible drawback of this -launching method is that, depending on your cluster setup, Chapel might have access to all physical cores on each -node participating in the run -- this will present problems if you are -scheduling jobs by-core and not by-node, since part of a node should be +``` + +The exact parameters of the job such as the maximum runtime and the requested memory can be specified with +Chapel environment variables. One possible drawback of this launching method is that, depending on your +cluster setup, Chapel might have access to all physical cores on each node participating in the run -- this +will present problems if you are scheduling jobs by-core and not by-node, since part of a node should be allocated to someone else's job. Note that on Compute Canada clusters this launching method works without problem. On these clusters -multi-locale Chapel is provided by `chapel-ofi` (for the OmniPath interconnect on Cedar) and `chapel-ucx` (for the -InfiniBand interconnect on Graham, Béluga, Narval) modules, so -- depending on the cluster -- you will load -Chapel using one of the two lines below: +multi-locale Chapel is provided by `chapel-ofi` (for the OmniPath interconnect on Cedar) and `chapel-ucx` (for +the InfiniBand interconnect on Graham, Béluga, Narval) modules, so -- depending on the cluster -- you will +load Chapel using one of the two lines below: -~~~ +```bash $ module load gcc chapel-ofi # for the OmniPath interconnect on Cedar cluster $ module load gcc chapel-ucx # for the InfiniBand interconnect on Graham, Béluga, Narval clusters -~~~ -{: .bash} +``` We can also launch multi-locale Chapel codes using the real executable `mybinary_real`. For example, for an interactive job you would type: -~~~ +```bash $ salloc --time=0:30:0 --nodes=4 --cpus-per-task=3 --mem-per-cpu=1000 --account=def-guest $ chpl --fast mycode.chpl -o mybinary $ srun ./mybinary_real -nl 4 # will run on four locales with max 3 cores per locale -~~~ -{: .bash} +``` -Production jobs would be launched with `sbatch` command and a Slurm launch -script as usual. +Production jobs would be launched with `sbatch` command and a Slurm launch script as usual. -For the rest of this class we assume that you have a working multi-locale -Chapel environment, whether provided by a Docker container or by multi-locale -Chapel on a physical HPC cluster. We will run all examples on four nodes with -three cores per node. +For the rest of this class we assume that you have a working multi-locale Chapel environment, whether provided +by a Docker container or by multi-locale Chapel on a physical HPC cluster. We will run all examples on four +nodes with three cores per node. # Intro to multi-locale code -Let us test our multi-locale Chapel environment by launching the following -code: +Let us test our multi-locale Chapel environment by launching the following code: -~~~ +```chpl writeln(Locales); -~~~ -{: .source} +``` This code will print the built-in global array `Locales`. Running it on four locales will produce -~~~ +```output LOCALE0 LOCALE1 LOCALE2 LOCALE3 -~~~ -{: .output} +``` -We want to run some code on each locale (node). For that, we can cycle through -locales: +We want to run some code on each locale (node). For that, we can cycle through locales: -~~~ +```chpl for loc in Locales do // this is still a serial program on loc do // run the next line on locale `loc` writeln("this locale is named ", here.name); -~~~ -{: .output} +``` This will produce -~~~ +```output this locale is named cdr544 this locale is named cdr552 this locale is named cdr556 this locale is named cdr692 -~~~ -{: .output} +``` -Here the built-in variable class `here` refers to the locale on which the code -is running, and `here.name` is its hostname. We started a serial `for` loop -cycling through all locales, and on each locale we printed its name, i.e., the -hostname of each node. This program ran in serial starting a task on each -locale only after completing the same task on the previous locale. Note the -order in which locales were listed. +Here the built-in variable class `here` refers to the locale on which the code is running, and `here.name` is +its hostname. We started a serial `for` loop cycling through all locales, and on each locale we printed its +name, i.e., the hostname of each node. This program ran in serial starting a task on each locale only after +completing the same task on the previous locale. Note the order in which locales were listed. -To run this code in parallel, starting four simultaneous tasks, one per locale, -we simply need to replace `for` with `forall`: +To run this code in parallel, starting four simultaneous tasks, one per locale, we simply need to replace +`for` with `forall`: -~~~ +```chpl forall loc in Locales do // now this is a parallel loop on loc do writeln("this locale is named ", here.name); -~~~ -{: .source} +``` -This starts four tasks in parallel, and the order in which the print statement -is executed depends on the runtime conditions and can change from run to run: +This starts four tasks in parallel, and the order in which the print statement is executed depends on the +runtime conditions and can change from run to run: -~~~ +```output this locale is named cdr544 this locale is named cdr692 this locale is named cdr556 this locale is named cdr552 -~~~ -{: .output} +``` -We can print few other attributes of each locale. Here it is actually useful to -revert to the serial loop `for` so that the print statements appear in order: +We can print few other attributes of each locale. Here it is actually useful to revert to the serial loop +`for` so that the print statements appear in order: -~~~ +```chpl use Memory.Diagnostics; for loc in Locales do on loc { @@ -170,10 +148,9 @@ for loc in Locales do writeln(" ...has ", here.physicalMemory(unit=MemUnits.GB, retType=real), " GB of memory"); writeln(" ...has ", here.maxTaskPar, " maximum parallelism"); } -~~~ -{: .source} +``` -~~~ +```output locale #0... ...is named: cdr544 ...has 3 processor cores @@ -194,11 +171,14 @@ locale #3... ...has 3 processor cores ...has 125.804 GB of memory ...has 3 maximum parallelism -~~~ -{: .output} - -Note that while Chapel correctly determines the number of cores available -inside our job on each node, and the maximum parallelism (which is the same as -the number of cores available!), it lists the total physical memory on each -node available to all running jobs which is not the same as the total memory -per node allocated to our job. +``` + +Note that while Chapel correctly determines the number of cores available inside our job on each node, and the +maximum parallelism (which is the same as the number of cores available!), it lists the total physical memory +on each node available to all running jobs which is not the same as the total memory per node allocated to our +job. + +::::::::::::::::::::::::::::::::::::: keypoints +- "Locale in Chapel is a shared-memory node on a cluster." +- "We can cycle in serial or parallel through all locales." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/_episodes/22-domains.md b/episodes/22-domains.md similarity index 68% rename from _episodes/22-domains.md rename to episodes/22-domains.md index c24ee31..95eaa94 100644 --- a/_episodes/22-domains.md +++ b/episodes/22-domains.md @@ -2,37 +2,34 @@ title: "Domains and data parallelism" teaching: 120 exercises: 60 -questions: +--- + +:::::::::::::::::::::::::::::::::::::: questions - "How do I store and manipulate data across multiple locales?" -objectives: +:::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: objectives - "First objective." -keypoints: -- "Domains are multi-dimensional sets of integer indices." -- "A domain can be defined on a single locale or distributed across many locales." -- "There are many predefined distribution method: block, cyclic, etc." -- "Arrays are defined on top of domains and inherit their distribution model." ---- +:::::::::::::::::::::::::::::::::::::::::::::::: # Domains and single-locale data parallelism -We start this section by recalling the definition of a range in Chapel. A range -is a 1D set of integer indices that can be bounded or infinite: +We start this section by recalling the definition of a range in Chapel. A range is a 1D set of integer indices +that can be bounded or infinite: -~~~ +```chpl var oneToTen: range = 1..10; // 1, 2, 3, ..., 10 var a = 1234, b = 5678; var aToB: range = a..b; // using variables var twoToTenByTwo: range(stridable=true) = 2..10 by 2; // 2, 4, 6, 8, 10 var oneToInf = 1.. ; // unbounded range -~~~ -{: .source} +``` -On the other hand, domains are multi-dimensional (including 1D) sets of integer -indices that are always bounded. To stress the difference between domain ranges -and domains, domain definitions always enclose their indices in curly brackets. -Ranges can be used to define a specific dimension of a domain: +On the other hand, domains are multi-dimensional (including 1D) sets of integer indices that are always +bounded. To stress the difference between domain ranges and domains, domain definitions always enclose their +indices in curly brackets. Ranges can be used to define a specific dimension of a domain: -~~~ +```chpl var domain1to10: domain(1) = {1..10}; // 1D domain from 1 to 10 defined using the range 1..10 var twoDimensions: domain(2) = {-2..2,0..2}; // 2D domain over a product of two ranges var thirdDim: range = 1..16; // a range @@ -43,31 +40,28 @@ writeln(); for (x,y) in twoDimensions { // can also cycle using explicit tuples (x,y) write("(", x, ", ", y, ")", ", "); } -~~~ -{: .source} +``` -Let us define an n^2 domain called `mesh`. It is defined by the single task in -our code and is therefore defined in memory on the same node (locale 0) where -this task is running. For each of n^2 mesh points, let us print out +Let us define an n^2 domain called `mesh`. It is defined by the single task in our code and is therefore +defined in memory on the same node (locale 0) where this task is running. For each of n^2 mesh points, let us +print out 1. `m.locale.id`, the ID of the locale holding that mesh point (should be 0) 2. `here.id`, the ID of the locale on which the code is running (should be 0) 3. `here.maxTaskPar`, the number of cores (max parallelism with 1 task/core) (should be 3) -**Note**: We already saw some of these variables/functions: numLocales, -Locales, here.id, here.name, here.numPUs(), here.physicalMemory(), -here.maxTaskPar. +**Note**: We already saw some of these variables/functions: numLocales, Locales, here.id, here.name, +here.numPUs(), here.physicalMemory(), here.maxTaskPar. -~~~ +```chpl config const n = 8; const mesh: domain(2) = {1..n, 1..n}; // a 2D domain defined in shared memory on a single locale forall m in mesh { // go in parallel through all n^2 mesh points writeln((m, m.locale.id, here.id, here.maxTaskPar)); } -~~~ -{: .source} +``` -~~~ +```output ((7, 1), 0, 0, 3) ((1, 1), 0, 0, 3) ((7, 2), 0, 0, 3) @@ -76,24 +70,22 @@ forall m in mesh { // go in parallel through all n^2 mesh points ((6, 6), 0, 0, 3) ((6, 7), 0, 0, 3) ((6, 8), 0, 0, 3) -~~~ -{: .output} +``` -Now we are going to learn two very important properties of Chapel domains. -First, domains can be used to define arrays of variables of any type on top of -them. For example, let us define an n^2 array of real numbers on top of `mesh`: +Now we are going to learn two very important properties of Chapel domains. First, domains can be used to +define arrays of variables of any type on top of them. For example, let us define an n^2 array of real numbers +on top of `mesh`: -~~~ +```chpl config const n = 8; const mesh: domain(2) = {1..n, 1..n}; // a 2D domain defined in shared memory on a single locale var T: [mesh] real; // a 2D array of reals defined in shared memory on a single locale (mapped onto this domain) forall t in T { // go in parallel through all n^2 elements of T writeln((t, t.locale.id)); } -~~~ -{: .source} +``` -~~~ +```output (0.0, 0) (0.0, 0) (0.0, 0) @@ -102,21 +94,18 @@ forall t in T { // go in parallel through all n^2 elements of T (0.0, 0) (0.0, 0) (0.0, 0) -~~~ -{: .output} +``` -By default, all n^2 array elements are set to zero, and all of them are defined -on the same locale as the underlying mesh. We can also cycle through all -indices of T by accessing its domain: +By default, all n^2 array elements are set to zero, and all of them are defined on the same locale as the +underlying mesh. We can also cycle through all indices of T by accessing its domain: -~~~ +```chpl forall idx in T.domain { writeln(idx, ' ', T(idx)); // idx is a tuple (i,j); also print the corresponding array element } -~~~ -{: .source} +``` -~~~ +```output (7, 1) 0.0 (1, 1) 0.0 (7, 2) 0.0 @@ -125,15 +114,13 @@ forall idx in T.domain { (6, 6) 0.0 (6, 7) 0.0 (6, 8) 0.0 -~~~ -{: .output} +``` -Since we use a parallel `forall` loop, the print statements appear in a random -runtime order. +Since we use a parallel `forall` loop, the print statements appear in a random runtime order. We can also define multiple arrays on the same domain: -~~~ +```chpl const grid = {1..100}; // 1D domain const alpha = 5; // some number var A, B, C: [grid] real; // local real-type arrays on this 1D domain @@ -141,32 +128,28 @@ B = 2; C = 3; forall (a,b,c) in zip(A,B,C) do // parallel loop a = b + alpha*c; // simple example of data parallelism on a single locale writeln(A); -~~~ -{: .source} +``` -The second important property of Chapel domains is that they can span multiple -locales (nodes). +The second important property of Chapel domains is that they can span multiple locales (nodes). ## Distributed domains Domains are fundamental Chapel concept for distributed-memory data parallelism. -Let us now define an n^2 distributed (over several locales) domain -`distributedMesh` mapped to locales in blocks. On top of this domain we define -a 2D block-distributed array A of strings mapped to locales in exactly the same -pattern as the underlying domain. Let us print out +Let us now define an n^2 distributed (over several locales) domain `distributedMesh` mapped to locales in +blocks. On top of this domain we define a 2D block-distributed array A of strings mapped to locales in exactly +the same pattern as the underlying domain. Let us print out 1. `a.locale.id`, the ID of the locale holding the element a of A 2. `here.name`, the name of the locale on which the code is running 3. `here.maxTaskPar`, the number of cores on the locale on which the code is running -Instead of printing these values to the screen, we will store this output -inside each element of A as a string -`a.locale.id:string + '-' + here.name + '-' + here.maxTaskPar:string`, adding a separator `' '` at the -end of each element. +Instead of printing these values to the screen, we will store this output inside each element of A as a string +`a.locale.id:string + '-' + here.name + '-' + here.maxTaskPar:string`, adding a separator `' '` at the end of +each element. -~~~ +```chpl use BlockDist; // use standard block distribution module to partition the domain into blocks config const n = 8; const mesh: domain(2) = {1..n, 1..n}; @@ -177,26 +160,22 @@ forall a in A { // go in parallel through all n^2 elements in A a = a.locale.id:string + '-' + here.name + '-' + here.maxTaskPar:string + ' '; } writeln(A); -~~~~ -{: .source} +``` -The syntax `boundingBox=mesh` tells the compiler that the outer edge of our -decomposition coincides exactly with the outer edge of our domain. -Alternatively, the outer decomposition layer could include an additional +The syntax `boundingBox=mesh` tells the compiler that the outer edge of our decomposition coincides exactly +with the outer edge of our domain. Alternatively, the outer decomposition layer could include an additional perimeter of *ghost points* if we specify -~~~ +```chpl const mesh: domain(2) = {1..n, 1..n}; const largerMesh: domain(2) dmapped Block(boundingBox=mesh) = {0..n+1,0..n+1}; -~~~~ -{: .source} +``` but let us not worry about this for now. -Running our code on four locales with three cores per locale produces the -following output: +Running our code on four locales with three cores per locale produces the following output: -~~~ +```output 0-cdr544-3 0-cdr544-3 0-cdr544-3 0-cdr544-3 1-cdr552-3 1-cdr552-3 1-cdr552-3 1-cdr552-3 0-cdr544-3 0-cdr544-3 0-cdr544-3 0-cdr544-3 1-cdr552-3 1-cdr552-3 1-cdr552-3 1-cdr552-3 0-cdr544-3 0-cdr544-3 0-cdr544-3 0-cdr544-3 1-cdr552-3 1-cdr552-3 1-cdr552-3 1-cdr552-3 @@ -205,78 +184,66 @@ following output: 2-cdr556-3 2-cdr556-3 2-cdr556-3 2-cdr556-3 3-cdr692-3 3-cdr692-3 3-cdr692-3 3-cdr692-3 2-cdr556-3 2-cdr556-3 2-cdr556-3 2-cdr556-3 3-cdr692-3 3-cdr692-3 3-cdr692-3 3-cdr692-3 2-cdr556-3 2-cdr556-3 2-cdr556-3 2-cdr556-3 3-cdr692-3 3-cdr692-3 3-cdr692-3 3-cdr692-3 -~~~ -{: .output} - -As we see, the domain `distributedMesh` (along with the string array `A` on top -of it) was decomposed into 2x2 blocks stored on the four nodes, respectively. -Equally important, for each element `a` of the array, the line of code filling -in that element ran on the same locale where that element was stored. In other -words, this code ran in parallel (`forall` loop) on four nodes, using up to -three cores on each node to fill in the corresponding array elements. Once the -parallel loop is finished, the `writeln` command runs on locale 0 gathering -remote elements from other locales and printing them to standard output. - -Now we can print the range of indices for each sub-domain by adding the -following to our code: +``` -~~~ +As we see, the domain `distributedMesh` (along with the string array `A` on top of it) was decomposed into 2x2 +blocks stored on the four nodes, respectively. Equally important, for each element `a` of the array, the line +of code filling in that element ran on the same locale where that element was stored. In other words, this +code ran in parallel (`forall` loop) on four nodes, using up to three cores on each node to fill in the +corresponding array elements. Once the parallel loop is finished, the `writeln` command runs on locale 0 +gathering remote elements from other locales and printing them to standard output. + +Now we can print the range of indices for each sub-domain by adding the following to our code: + +```chpl for loc in Locales { on loc { writeln(A.localSubdomain()); } } -~~~ -{: .source} +``` On 4 locales we should get: -~~~ +```output {1..4, 1..4} {1..4, 5..8} {5..8, 1..4} {5..8, 5..8} -~~~ -{: .output} +``` Let us count the number of threads by adding the following to our code: -~~~ +```chpl var counter = 0; forall a in A with (+ reduce counter) { // go in parallel through all n^2 elements counter = 1; } writeln("actual number of threads = ", counter); -~~~ -{: .source} +``` -If `n=8` in our code is sufficiently large, there are enough array elements per -node (8*8/4 = 16 in our case) to fully utilise all three available cores on -each node, so our output should be +If `n=8` in our code is sufficiently large, there are enough array elements per node (8*8/4 = 16 in our case) +to fully utilise all three available cores on each node, so our output should be -~~~ +```output actual number of threads = 12 -~~~ -{: .output} +``` -Try reducing the array size `n` to see if that changes the output (fewer tasks -per locale), e.g., setting n=3. Also try increasing the array size to n=20 and -study the output. Does the output make sense? +Try reducing the array size `n` to see if that changes the output (fewer tasks per locale), e.g., setting +n=3. Also try increasing the array size to n=20 and study the output. Does the output make sense? -So far we looked at the block distribution `BlockDist`. It will distribute a 2D -domain among nodes either using 1D or 2D decomposition (in our example it was -2D decomposition 2x2), depending on the domain size and the number of nodes. +So far we looked at the block distribution `BlockDist`. It will distribute a 2D domain among nodes either +using 1D or 2D decomposition (in our example it was 2D decomposition 2x2), depending on the domain size and +the number of nodes. -Let us take a look at another standard module for domain partitioning onto -locales, called CyclicDist. For each element of the array we will print out -again +Let us take a look at another standard module for domain partitioning onto locales, called CyclicDist. For +each element of the array we will print out again 1. `a.locale.id`, the ID of the locale holding the element a of A 2. `here.name`, the name of the locale on which the code is running -3. `here.maxTaskPar`, the number of cores on the locale on which the code is - running +3. `here.maxTaskPar`, the number of cores on the locale on which the code is running -~~~ +```chpl use CyclicDist; // elements are sent to locales in a round-robin pattern config const n = 8; const mesh: domain(2) = {1..n, 1..n}; // a 2D domain defined in shared memory on a single locale @@ -286,10 +253,9 @@ forall a in A2 { a = a.locale.id:string + '-' + here.name + '-' + here.maxTaskPar:string + ' '; } writeln(A2); -~~~ -{: .source} +``` -~~~ +```output 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 @@ -298,52 +264,44 @@ writeln(A2); 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 0-cdr544-3 1-cdr552-3 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 2-cdr556-3 3-cdr692-3 -~~~ -{: .output} +``` -As the name `CyclicDist` suggests, the domain was mapped to locales in a -cyclic, round-robin pattern. We can also print the range of indices for each -sub-domain by adding the following to our code: +As the name `CyclicDist` suggests, the domain was mapped to locales in a cyclic, round-robin pattern. We can +also print the range of indices for each sub-domain by adding the following to our code: -~~~ +```chpl for loc in Locales { on loc { writeln(A2.localSubdomain()); } } -~~~ -{: .source} +``` -~~~ +```output {1..7 by 2, 1..7 by 2} {1..7 by 2, 2..8 by 2} {2..8 by 2, 1..7 by 2} {2..8 by 2, 2..8 by 2} -~~~ -{: .output} +``` -In addition to BlockDist and CyclicDist, Chapel has several other predefined -distributions: BlockCycDist, ReplicatedDist, DimensionalDist2D, ReplicatedDim, -BlockCycDim — for details please see -http://chapel.cray.com/docs/1.12/modules/distributions.html. +In addition to BlockDist and CyclicDist, Chapel has several other predefined distributions: BlockCycDist, +ReplicatedDist, DimensionalDist2D, ReplicatedDim, BlockCycDim — for details please see +https://chapel-lang.org/docs/primers/distributions.html. ## Diffusion solver on distributed domains -Now let us use distributed domains to write a parallel version of our original -diffusion solver code: +Now let us use distributed domains to write a parallel version of our original diffusion solver code: -~~~ +```chpl use BlockDist; config const n = 8; const mesh: domain(2) = {1..n, 1..n}; // local 2D n^2 domain -~~~ -{: .source} +``` -We will add a larger (n+2)^2 block-distributed domain `largerMesh` with a layer -of *ghost points* on *perimeter locales*, and define a temperature array T on -top of it, by adding the following to our code: +We will add a larger (n+2)^2 block-distributed domain `largerMesh` with a layer of *ghost points* on +*perimeter locales*, and define a temperature array T on top of it, by adding the following to our code: -~~~ +```chpl const largerMesh: domain(2) dmapped Block(boundingBox=mesh) = {0..n+1, 0..n+1}; var T: [largerMesh] real; // a block-distributed array of temperatures forall (i,j) in T.domain[1..n,1..n] { @@ -352,12 +310,10 @@ forall (i,j) in T.domain[1..n,1..n] { T[i,j] = exp(-((x-0.5)**2 + (y-0.5)**2) / 0.01); // narrow Gaussian peak } writeln(T); -~~~ -{: .source} +``` -Here we initialised an initial Gaussian temperature peak in the middle of the -mesh. As we evolve our solution in time, this peak should diffuse slowly over -the rest of the domain. +Here we initialised an initial Gaussian temperature peak in the middle of the mesh. As we evolve our solution +in time, this peak should diffuse slowly over the rest of the domain. > ## Question > @@ -368,13 +324,10 @@ the rest of the domain. > > The first one will run on multiple locales in parallel, whereas the > > second will run in parallel via multiple threads on locale 0 only, since > > "mesh" is defined on locale 0. -> > {: .source} -> {: .solution} -{: .challenge} The code above will print the initial temperature distribution: -~~~ +```output 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.36954e-17 2.79367e-13 1.44716e-10 3.29371e-09 3.29371e-09 1.44716e-10 2.79367e-13 2.36954e-17 0.0 0.0 2.79367e-13 3.29371e-09 1.70619e-06 3.88326e-05 3.88326e-05 1.70619e-06 3.29371e-09 2.79367e-13 0.0 @@ -385,23 +338,21 @@ The code above will print the initial temperature distribution: 0.0 2.79367e-13 3.29371e-09 1.70619e-06 3.88326e-05 3.88326e-05 1.70619e-06 3.29371e-09 2.79367e-13 0.0 0.0 2.36954e-17 2.79367e-13 1.44716e-10 3.29371e-09 3.29371e-09 1.44716e-10 2.79367e-13 2.36954e-17 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -~~~ -{: .output} +``` -Let us define an array of strings `nodeID` with the same distribution over -locales as T, by adding the following to our code: +Let us define an array of strings `nodeID` with the same distribution over locales as T, by adding the +following to our code: -~~~ +```chpl var nodeID: [largerMesh] string; forall m in nodeID do m = here.id:string; writeln(nodeID); -~~~ -{: .source} +``` The outer perimeter in the partition below are the *ghost points*: -~~~ +```output 0 0 0 0 0 1 1 1 1 1 0 0 0 0 0 1 1 1 1 1 0 0 0 0 0 1 1 1 1 1 @@ -412,56 +363,57 @@ The outer perimeter in the partition below are the *ghost points*: 2 2 2 2 2 3 3 3 3 3 2 2 2 2 2 3 3 3 3 3 2 2 2 2 2 3 3 3 3 3 -~~~ -{: .output} +``` -> ## Exercise 3 -> -> In addition to here.id, also print the ID of the locale holding that value. -> Is it the same or different from here.id? -> -> > ## Solution -> > -> > Something along the lines: -> > `m = here.id:string + '-' + m.locale.id:string` -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 3: Can you do it? + +In addition to here.id, also print the ID of the locale holding that value. Is it the same or different from here.id? -Now we implement the parallel solver, by adding the following to our code -(*contains a mistake on purpose!*): +:::::::::::::::::::::::: solution -~~~ +Something along the lines: `m = here.id:string + '-' + m.locale.id:string` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +Now we implement the parallel solver, by adding the following to our code (*contains a mistake on purpose!*): + +```chpl var Tnew: [largerMesh] real; for step in 1..5 { // time-stepping forall (i,j) in mesh do Tnew[i,j] = (T[i-1,j] + T[i+1,j] + T[i,j-1] + T[i,j+1]) / 4; T[mesh] = Tnew[mesh]; // uses parallel forall underneath } -~~~ -{: .source} +``` -> ## Exercise 4 -> -> Can anyone see a mistake in the last code? -> -> > ## Solution -> > -> > It should be -> > -> > `forall (i,j) in Tnew.domain[1..n,1..n] do` -> > -> > instead of -> > -> > `forall (i,j) in mesh do` -> > -> > as the last one will likely run in parallel via threads only on locale 0, -> > whereas the former will run on multiple locales in parallel. -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 4: Can you do it? + +Can anyone spot a mistake in the last code? + +:::::::::::::::::::::::: solution + +It should be + +`forall (i,j) in Tnew.domain[1..n,1..n] do` + +instead of + +`forall (i,j) in mesh do` + +as the last one will likely run in parallel via threads only on locale 0, whereas the former will run on +multiple locales in parallel. + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: Here is the final version of the entire code: -~~~ +```chpl use BlockDist; config const n = 8; const mesh: domain(2) = {1..n,1..n}; @@ -479,51 +431,50 @@ for step in 1..5 { T = Tnew; writeln((step,T[n/2,n/2],T[1,1])); } -~~~ -{: .source} +``` -This is the entire parallel solver! Note that we implemented an open boundary: -T on *ghost points* is always 0. Let us add some printout and also compute the -total energy on the mesh, by adding the following to our code: +This is the entire parallel solver! Note that we implemented an open boundary: T on *ghost points* is +always 0. Let us add some printout and also compute the total energy on the mesh, by adding the following to +our code: -~~~ +```chpl writeln((step, T[n/2,n/2], T[2,2])); var total: real = 0; forall (i,j) in mesh with (+ reduce total) do total += T[i,j]; writeln("total = ", total); -~~~ -{: .source} +``` -Notice how the total energy decreases in time with the open boundary -conditions, as the energy is leaving the system. +Notice how the total energy decreases in time with the open boundary conditions, as the energy is leaving the +system. -> ## Exercise 5 -> -> Write a code to print how the finite-difference stencil [i,j], [i-1,j], -> [i+1,j], [i,j-1], [i,j+1] is distributed among nodes, and compare that to the -> ID of the node where T[i,i] is computed. -> -> > ## Solution -> > -> > Here is one possible solution examining the locality of the -> finite-difference stencil: -> > -> > ~~~ -> > var nodeID: [largerMesh] string = 'empty'; -> > forall (i,j) in nodeID.domain[1..n,1..n] do -> > nodeID[i,j] = here.id:string + nodeID[i,j].locale.id:string + nodeID[i-1,j].locale.id:string + -> > nodeID[i+1,j].locale.id:string + nodeID[i,j-1].locale.id:string + nodeID[i,j+1].locale.id:string + ' '; -> > writeln(nodeID); -> > ~~~ -> > {: .source} -> {: .solution} -{: .challenge} - -This produced the following output clearly showing the *ghost points* and the -stencil distribution for each mesh point: - -~~~ + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 5: Can you do it? + +Write a code to print how the finite-difference stencil [i,j], [i-1,j], [i+1,j], [i,j-1], [i,j+1] is +distributed among nodes, and compare that to the ID of the node where T[i,i] is computed. + +:::::::::::::::::::::::: solution + +Here is one possible solution examining the locality of the finite-difference stencil: + +```chpl +var nodeID: [largerMesh] string = 'empty'; +forall (i,j) in nodeID.domain[1..n,1..n] do + nodeID[i,j] = here.id:string + nodeID[i,j].locale.id:string + nodeID[i-1,j].locale.id:string + + nodeID[i+1,j].locale.id:string + nodeID[i,j-1].locale.id:string + nodeID[i,j+1].locale.id:string + ' '; +writeln(nodeID); +``` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + +This produced the following output clearly showing the *ghost points* and the stencil distribution for each +mesh point: + +```output empty empty empty empty empty empty empty empty empty empty empty 000000 000000 000000 000001 111101 111111 111111 111111 empty empty 000000 000000 000000 000001 111101 111111 111111 111111 empty @@ -534,26 +485,22 @@ empty 222222 222222 222222 222223 333323 333333 333333 333333 em empty 222222 222222 222222 222223 333323 333333 333333 333333 empty empty 222222 222222 222222 222223 333323 333333 333333 333333 empty empty empty empty empty empty empty empty empty empty empty -~~~ -{: .output} +``` -Note that T[i,j] is always computed on the same node where that element is -stored, which makes sense. +Note that T[i,j] is always computed on the same node where that element is stored, which makes sense. ## Periodic boundary conditions -Now let us modify the previous parallel solver to include periodic BCs. At the -beginning of each time step we need to set elements on the *ghost points* to -their respective values on the *opposite ends*, by adding the following to our -code: +Now let us modify the previous parallel solver to include periodic BCs. At the beginning of each time step we +need to set elements on the *ghost points* to their respective values on the *opposite ends*, by adding the +following to our code: -~~~ +```chpl T[0,1..n] = T[n,1..n]; // periodic boundaries on all four sides; these will run via parallel forall T[n+1,1..n] = T[1,1..n]; T[1..n,0] = T[1..n,n]; T[1..n,n+1] = T[1..n,1]; -~~~ -{: .source} +``` Now total energy should be conserved, as nothing leaves the domain. @@ -569,22 +516,26 @@ Let us write the final solution to disk. There are several caveats: We'll add the following to our code to write ASCII: -~~~ +```chpl use IO; var myFile = open("output.dat", iomode.cw); // open the file for writing var myWritingChannel = myFile.writer(); // create a writing channel starting at file offset 0 myWritingChannel.write(T); // write the array myWritingChannel.close(); // close the channel -~~~ -{: .source} +``` -Run the code and check the file *output.dat*: it should contain the array T -after 5 steps in ASCII. +Run the code and check the file *output.dat*: it should contain the array T after 5 steps in ASCII. -# Ideas for future topics or homework + -* binary I/O -* write/read NetCDF from Chapel by calling a C/C++ function -* take a simple non-linear problem, linearise it, implement a parallel - multi-locale linear solver entirely in Chapel + + + + +::::::::::::::::::::::::::::::::::::: keypoints +- "Domains are multi-dimensional sets of integer indices." +- "A domain can be defined on a single locale or distributed across many locales." +- "There are many predefined distribution method: block, cyclic, etc." +- "Arrays are defined on top of domains and inherit their distribution model." +:::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/fig/.gitkeep b/fig/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/files/.gitkeep b/files/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/files/linux-cloud.jpg b/files/linux-cloud.jpg deleted file mode 100644 index 9f573bf2511b1fba578d0f84217d897956a3ba42..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24843 zcmbTdXH-*P^euW)Ab=1A0tN^*D4~dS5^4~oDIEj>6{Q!E(2HmQLB-H}3=pL^1*Ibj z3ItS&0TF3Nsz~pOh%dkYz2m+y-lzB8$;gL{oa|)n?7j9}bI$c|>>mR-Zfa;^2!KE! zz=in${v82Y2G-~GZ7r-d#Xa3l`vkaqiiavGD2ii3g1r4b#Z63!;F8;OUiE6d1> z6GE>0c?U;`>xgTpar~PA3;;G(Rv0S_8w>_xXJ_N!L~wB)JH{!<%g2on5fT#>5kjHF zr4*#aCFRj56h=cvUQtO^RaHzHr-N12R!~t@{%;TvJ3Bk)F-{~G7gAXQC87NP`TExZ z@UVfrz(fd05&-jnAUvRdU4R$>fLNHJ{uk~4zCd6Il!X<>#?En!c|*f-01Sdaz)%Pa z3lz${JDT}E0Oeufl~B@W<+HjDlO!ONW73|mq0iQJ@>_q}kW#r37|YHfZ~`ePbn=um zMn+auO;FXd|A>o+85bA|g+O8d#RUR~F`p0~D2s#=E3dv4 z>^gx@QaOeVaW?HqT_-zQ#d?GPM&Kt70V&n*CpZ5K?SCTse-~Kn|F6jYPhkIdT;l*I z1jPLFAUuE`uy=kY=e0Mtqn0ITlsp9+%?V5{&0VPur-BVKz9jnJvBa#JOcMc2$9@Sl z6$4`H2MC4?lMdB9=nN6TX%ps}{0}g-n$8_;IJjdAly{NR`gTj~I<%5hJ+=Czhpg+% zCN`)9Q+j-TF_^qVLd7rFYT2BJQ-1n6F^2VP*c)4FAqCfZQe?hY;hlmVfj{V)zJJwR<``$1nQQZbB{X+YTU3iaC%V%SpyM?S>Vy@V z0vsD`O9#U-gOUE0pbV`v6#&>Z_Ye8HU*YlK>Jq0FTDN(5fv<-IBx>bPaIE$KTLQ z50MLO(}{3z*7MSkf)P^34I)o&6+HY&ZcMjY;v3T>hi73ti9o7J=TxeXfPdL^+5^-u zmJoyIQ7b-1bM)1$jCWq^HfUSy6w3$n!Y*!_IwhBFsPWva)^M?O6E;Ep#enFF z&bPZ3N2zx#V5$YzalL5zlk}T;B0s>^=M7D3f9lCr@1Ml?RX;A|x$OAfphfpdL^lu& ziY3BSQGJq)#=ucXe+n;q!s+T~Qsyn}7o;DYME1uIS$*@GA!^>wAk1Vc9qUvZxE!8M z_XZok=(#(Atj+}J=KKf&?> zHfv|sp+AygO7a4azd@y4NV7B)Mm!tq*rl>JCQZCF(^;zg*=%j@O)EGLs1?qEAB=ex z(#f`DOQlhwv<>YPd2)t~gpaHYf$1KkAmb?vsQTS#ZbUaUtc1CqK&$45O6MiIfz!HQ zj`Sq{0nZ*C{*JmAMgpH%%Z=j(xaCm2F-Q6k(4-^BjC!pO~&JT5p566 z^Os{4En96pGe+yk{7GiC2N}OCRBDxie%VQ$Rx$ z3wCz1Qo|w-{5Xd*D^P`Gq9~{*ysucCFZowlMvP_A?LQA0k(L0fmTl2mT;jvLpl&Rd zgB1gsg;8w6g1Tk5a-rOB^O~f^XNaf^RXj1hvke%Vm;&xEs>K4dvn-Mn5(v+pvW3C+ zaQg(9N1V!}-qCXeZn5b(CM8>3-8o#Bh91XEq+DFo!GJUOfWNLt)7ISeAW zLxp*e=5EA{#>-Lo@W&<)!BkQ(?umu-5NJ9R?#&?a75e)~j@|JB83`hizh1Qfr%hCV zi=122!$mUz#w)At7+B&GOD4 zlt;Bw_@x1{X=j0KL#@ZdYBi{<=Ufa~r+1}D7cUD+PZRlh#EI3u@igSjlMwq9&X#^z z6-~xo$+12J=4CmIiK|q34lh43OQ$zQU;$OHoi@3e?FhFfc;ZE{DBxwJ$89fH6p&abCBN|r_-+& zonXfwtS!BiFhR4Q z-Y7-*c{_lm!|Pj$5P(Mpt_U%Tq^SajF%}KDSND%v&umvNm=%qdymYxG{y$sw2gKo! zx-U3OPI|eto*3a8V{C5!WEs!J+&!U1dhr?ogJ>~RK#;ijdn8J6P8#K@`r`OC!NCT# z4Y4Oncn)W|*sctEwh3Yb*RPl~8AFOBUtH~gFsF-;;#q>3w@`(8EScIqXC{jH7t}09 z)NZl7asXUMMrlxWrQC0ZD0DBKonHc02GM<;B_{pc^hyU`EIX|R(2|XiBXOiiF&5Gy z0!=Tf~|b|zisMD_9^Yun4FcdlwK(us&H2u|<2xj}=8YHr6n=J3Uf zL@oeVtWC0UqPV6R2MrL`$?2-&f#&*D)|8gvQ<-S~L4$RRaR{Lsma$(B2xnJ|lalLio~$s3VJnAC z4J7{oAlG!Ac)%Q7_nx%)TvY(@+^LKxu%9gQbC{j3k@Ah{2ekP{JTm!`Bk3y8tKLtV z2WtRr>*Oe#sNoB#W1ZcAu`xujRMX#?PH`EL?=Qq-Kog$;uEFn?*(|S1(~3QNkc(pk zOQrEZi&;FMR(_8*@%3jh4no>RYovf6WMsszBiai@H?CY!XC!~Ed|_TSdE7fkg;*AZ z*%h}}i-nxsnEN7fvglM|51*ZmAhu*ZYWnv>fZRjk&{|Do580D9)%>#0g z7>Op5)CD32Qh5KJ_Doa-cdZ&h8|TZLT5WD*GurUjf8gGT0TlsC(!1YO!DDVe)hhHp z?N;#b4R$w(SR4NrQ+-HNjdg1SUxzK2oq-x+J(;k2=a&gq*>QqrEA4apa^*=YqcRt{ zeEQR)zTA%D_|V_&{lW3JseoubkZ6$ntNizL;3P*bCxMCm61q+4q zwbDLE!S@L$h9$&-LAcoGnl!R;pOV(E6uaoHJp{FR`=Y2TQm8-a^*hCjAwQyhufIOu zOMC6BaKtBhf;ROdnwhZPBi+KvXzGJKbt|Ql>c_txd`|x=PGMNG*5+n)o2m@zWx03b zQsF)`CQY$iz~xJwf`^q}?@6nNL3YO%>#A5@U*w4!5dv~>HzI9brrGVQYm#gkhp|M? z84COIcwX}zRo=6Ih9Fa!DVag*)@ZPzg&G?Sm6(Pg6=Z@=83pRz8@V?}vON<=in(}6 zqII8o3_6p$qA5gDl%@=gNhbmql~8b5C%s=sP58h ziRTR6I4_PB@vaUcYLWp6gO?4qY@y!<6D(JUh6w<0*v;wG4Q~U5 z$*AG5AUub~Xoj|R&~y%zbAc$Ct*7Xsb*2K*)o}6f8+K4oO;d!$ZA>)f*rR0oX~+PZF0dT`5b> zoM~}%8J7Yjdx0OljSWeKN4oM?mv zV<*XBIfG321>x_*in|QEft+HxO@C+*=(#^vf{4-OU`eBPMddYVR3bq*!kyM|Zr+^G z4XAyqZ~^wz9P#c}&0H<(Hp&WgF1!V^mVoe2j$h>=RcPOMR!uE`U*IERb_sJXruI)W zwgNrtC9Tao>_M@qMxt5Q-QLa>4pb_G^`#SSv$Q*k=RkC#D)09MKw9^+yNOI6Z29t! zS+;ld>WAv2BUMn4p7;VKUF>ZY`h;bqO%u;h%I!p;{<0yvnsWRdeSRx^hG5yb%cx`} zJfF(0ltyET_W-=!;Ec!OPh?g->8=x|A3b$3`h=81wk#eM?IWWZ{-q+bKwa4KjS z4uYryoSPIuo@K-iWD?T6gIZao+^k-3{n^-&Hb@< z{WeaN?DI=NmZh8iowbm)i&!>(l1SkI9*{^ZWzf}voh=bU%xE_Dv7nSC^yabJ_gP%i z6jlVJFp{G1$%JU(xuD}i_cVsdK!K<-o^M4DG8=}^1AZa@*>#vOWhoAl7bSG+&QJSgeGfE1&>&qF`O z!lNufhQ=ku+DxRLk;wknYjyCN<3+w-9Dq!bZehMwgW{ICi7JTC-Quc6iJ`@ZzJ>fW z5;>_?hYQfa8g|Z&M4yOd-&I}EvW-iO!!Z{Li$d3yk9umXzUpg_H6=R_?2RBiSD2mD zy0As$q8j`8ff`=f*z$;dwIoa@7pI~sq|hXa5HJ>>WpeIH^ZK%Nf}cx4 z{n)S%hC}DCk9q*(`A0TY(9@#%pE81(VU^x!|DFKZ4td*I$`G^#J7{|TY?s^qLf)GN zDGJEiO*y^n6p`m;8wayQJ>lhFq-T0ZFabGIjxdTFB|=*6o6Bs9FDFJ;#{Z5SX$viN zVh$o~v>#E%qC-d|AaTV%gYazc&?U0T7#*3K&K?#(UvY|JcN>E$q}SYLiL$7RXHUqzjet|M#3{xwUg;4ruX{otOsXF)yahPwRWbJ6z7amr=^ z?4lha=zS#v{;u8ezFZ$v2&ipA4UZ^k)44w8xgdu#XY+HZ>>!#!qVs$ZWyGgziv^X? zryXetIpDC`+8%%0$(qTt-VJDlI_W+wy{aesz!k9RBBm?h;Xr|RM^L{KN>t|e0#V5P z{N>t!GdFwm=AZrpG^@w#aAiwNtw&RDmMWRAfh7yrXey_uOh0Kyb#fce(|q#4_M$^v zkL+l8xkGvvXk>`xcKSCBM6)b64)$D4CM5vMD6CO5N}z31Sm`J4otG&+8w!Sb&lvpiRHH?Gqz>1G z`T!Zzxga`GeEpXttIR;NAYvR~$1RzvEcFAk6uxmX`y^b;5}X>fQ@V5LJ^unwrNfye zC#f@l{m?tnL9Yg_bh>Ybaxd`XI~#Nq%qZ^9q?GTX9->u=bu!v zRm~OBx-uIipra!@-T2;*qX}8QQ(=3#>lKBoiwrbw@zbEL^t|R-`M{ul(|J_7O8w4k z$@zx;t)g4Yc-}4z)ST_C_$>3&m4(y9N@HokDx8GrqW3-(#($>pPJ{C5`3@Mi{ApT| z^DZkzq;NfKhssXu#1P-iU?6 zSpcOXP@t@0ag$H2@`<#IV=aTGz=N3S+&RFUeUD!*euCymA`IvW0r5rRth+1TL@->x zN**twV)sDd&r6~f2T;Bl8_ICm?YEcW&V=tq`nl>Tec>-epPG92yc7ej8Ixp3y4)^P zODdWye5vimSd-%QIjXtK(opoU_*m9={kbnEuX7Dh7vw@Fgw?||oGn>b2Om)5$w>X| z>$DHO{Z>NBb>XI!)n1yjH8+>W5dw8bn;`!cIenMJQ(fLEw=G#Yh5Sn}ylNv=X&n&6 zZ+Mw!g7SOQf55Tir>!Tg2^4Y!34t@~AO7bh@Uq8 zfjb0^d{)2?y|;sIl!3A(m?&&!;UH|XQJiPmXd4jSI!IcVY9A3;EO>+PK`DTEqsg&SvF>l<9unuJF zEaOXCCj;K#3?HHiqA1-^#+V8HhlpZr>-u;?j5#AUvLLf0oXYm`x`o_$CM5a%1=1YZ z%ccVX#6(w286jj-y)8I_B|HdqgTyUQ2`;g@IB-I`@RlPGpOt%4gv&Iw+6ybVTM;f! zprsIWWL|IwXte!>sbS+AfAMmL8Zqt6)>QJuiXmR61fa>nXMrR zut1KQDV8%^XMz<$!CoC%bxXF26w~X1F!2HmD7hJB!#!FsCQaHYKk#H?{M0FL6YLz_ zB^x%LNg`Q7<#j!l%ycYFvocYeR>t5V=(!A{nz4J4;@8d@Ix{=#og%`vyVIu7qNLBDj!v*vt$AhW_Mh4lh zX|vk+wV%&`u9ZZGs*z%be7f8$siK$4&Z{D(^}=N7Qb5ozyO2qSz-|>$00p0}2OhkU zIo(?(d;Nik%w~alvY8f(A)w8q8ztNgzvB;m=(;eugAv7Z*xQ($nPPYGzZ(LKRLG+UsuW{RWR1OCXF0y>0dt zLVX@SQtEJ#`S|q*7hdY!5JcM0u2-t=Cs4_SJU{)_KZ52+51bPabdIzOd-n{t_{pK5aDJ5|ZL|{?7L(_}I6UxOtCd=gf44AA6oRXrFUpdZsmR#1!C0k!fN+`PD(| zc!2@_fi)Sy1Xh`fQ}&!uHu^|uy?8*Y?tsqM(jgVIYEqDp-|-`Y%Fe3NL^&Y8L|%D? zI`~XTo)aoKO+^U-sjEH8`2C7%;Ex|nZ|gJR_XttYU>XJBH_90`<3-a0~L0(e`>B2~4_-S96Ic+Lk@XfR_gmkp;cQ0+J+X9v% zmBki~txhA-S^Sb-&HV#*%OMROO3$aH4y@y4->7RuM%ounhqwCnOY(RJ)Lt{D1^Yxm zL=1h)Hrj_Le|##Y*VED0^IHp4@WNWJ31k8zOys8^vizix+QOw1Uq;{XJmc!_TJ={D zo{tdbC=M0P1pSl^>cFYC5t-}a$D}>k`;K1wo6X`-mv?)DB2O8?;tbTNSW-jV!`G5y zj`CYB?`|AUB)8VGz*vTB9pgJn-Qta+3Iu3s6>wKp)cvbJn^5OYoERu z=3d=@Q>3!lq3DMCW=3Jp^IvDbb&*%&CL7yjf2$xjrTYtl@S@rFFmXDfuT$_;I+BdF z#+_8bqFNXkCIn@tPN^+LcA@MW(XdlQQ3Ut*A5rwxfx9K)+IM+E*bjY0pMFUaiG7nT zA$3otfP1uUT_AjRWjv%PJTfZkQB~f(%LAc=rC&p^V4xej(BJ(nJOJ-zL5FH z2FZxS!_3S0If*YrRt=O~V^wy`AEl*ZKvK?_#l%>n@gj(5OWDiAqEmU8-|RZvHGq9? z`S*u&gv3g^8;R33vEb6Bi>ye<+Dv7tF%8K??tJsP*?p(TFMw|2j^%!xG0wEHuqEfU zy2z^@O@@Am51B`{po-If$tFz$DI$Bj)%nr#9>e`zLC-d@=;!k$ML!CC>b=*6{ci1} zZIW4xs(u`Up4PdaYbx@wjFJf4KmV!hb)Cr?8-NlI8RUO?(|ylG#Eyrj)o-_iu4XL# zf&`S0jqCrAol~y)nDkOyVmuHl%|u?}GbMH(z;ISQM?#7>&3VZ6N{kaTxtJV>2w#AF zORO(<4{%`4A3o6_piH{m`Ju3u8E`QzS&>zzq8k<>68^-5S^Qc@T7r{{c}8<#r}CQx z5ci4ZULa)+YYp+8(sksZY0RRVG*hbBG6wC4nbt_TS(6E-R ze+94K?$r6o$vd^Vj%Q!sb}TCYW^hX&sHJXL^-q(Rr1uGV{13UDbQ6zE!GYQIvpd~@ zSy`3Nr?jEG%S|qO{B(?qp~~Q?mn%C^8T1hd-%d&p}ZRLL?FdKYVQ1{hZ;3{5k>Az|S_=*mk`|Pc3IV-qd z&K*hH@^T@-%p2Uio2hNM^2B6De`UiB7@qVRICNx z)ZBAb9Zjw8a{VaRyHACm`JVOBOUfWISPY3b&cKI%6t3k#Jd+V34*OHhhC7pjZ-gg-YWel606yj zU??(CIFR$M{Q1o9I#G?6CQTxUJzQ0z9+&E}hJ4fh&E`daLgLSARV^LG`s^RKzbu#t zXW!}}V#Sb4Jo@FbV@GR7SGwTpFHvq{i&qxzyNW)1I^on8EI8(#KaqVyE@Km=q3l~S zs#B@7U5n-5t}&P~vFW;V@>Bhv3r$M$F$LJc4>2QTV8;Y`bLq?ItFRe4vyxvqtBQG& zCzQYr!5z=t{E?gVm%FFAEk-Y1^KZO-E}}zs2Q)hle-w1${^KsE(27)L&u1{)VYgd9 zc==#D{r>IYN$1{S*%0C6yRrvR@4bg*Njeg?_3ZNxIv5=9hHmqf{)$|?P=0wKE9E^M zZW(0>!)Pms#cO+)`dkWiUzN6y?=@dPJ^1|lWyh>n(@MWePCm(c^?==0?vO+Cs9}ZA z>Gwv20edXR9N}`LMST5~uod}e4A-2=FW*1JN3n#R{C-ncL8BE;-1T ziP47J?7l%I-RqK5} zo@RYY^KqAmV9Jz#Gq^=0$Tq57?EH&lwweav3n{XeuH#<^VQT9}<$vXbFK-Ow*EmJ+ z^&J0QbtaeS--}zuqhMod5_kQyT~#zcbcHE3y?=Axr|xOEtMT3AKhusax$q4BX)xJr z7TUQmr8!~y!dI)j>FkHN)rUaJBWW{5m=3SD?u;C?kIG2=U>`|z&7v4^t$#1YQROj*bk9z>4SkF{{j%E9gNLz>4~ zF<=>XzuAVfMz}@MT?s#bfRwRp)!RZF{ACHXK}m@J5h_uUjO#u0>7S5NAE2ub#Yue; zJ~{H`g~0Eatk>pc$C#AOn^Q%;yeg_BC#>-Kz^1_Hr$=8u6HPqEe$R>r^#^bMxRmzv zncyT4maRQ1@Z4?hi}4RzXezd^=T0VQR3Da&AT$^PqFGXj>^S}UM28Yy{4>3W`em3K zk)%jVn40(Zj-Mw}kyCEcUjA%@CAai_`CI)+*MkZgd_7nv-9pN8!!Y8OESC2Pznd#Y z717S^th}s2)_}>SK_HmQZl#P>QzqT+eo0PpBJt1~WJS$EV4@V`(*%NUQC@r*gqpL2 zY}dPml#d%EXC_}zFp|{*nL!&36=BaKfdms{9D#P-_G6C;DzjtN+WBI20GwbM*L=H~ z3R4|J)flV9kndp_*|YSL2!2DnDo{)eFjMehxnVxTzEfeKpv+8?8PcPr z!!CPBe-IQc)A7mAykRJqA|6ar^=(=Zj%{Ya4-iIQuB@raKqAB|n`FUzRCsm36FjZN z>EWw-2+CgSi(~)uewj;x48@vgJqKQqCfOUQ$)F!z5br{w7X69TOqJB*x3m9%&o**( zF8ndOsvyYc zszhae}1}SfW1HbTj%Gz+xmAjnzAKu7dP|gIO=A({FxHT(J2c~FPrO8eOgcu{u&e^ZWr+bdq@)C9rL$*V0ZcR6-+yq~K*zh5`V z-`CWvB;VbM&Jsj3GwxVrMlvDA@e=Y#V*Q$4_%*Z56QuaR6TW;IxAyJkZa%x)8c`F? zH^Z`c|9b0-mc+~BYVSL>=DHYLEjs6c$lcFr?M>Ozft8GB44h!4BX|6Yw$4I(ON9T) zKf@+{sJ)%as{*6i`gbLtJufa+{2*4yIn?ZO;!knp7gL%y`niY~xVW|FF}#%Q_obE= zCb{mAS8H6c_KZ5bTClKiE}89%!b$*3_+l64WFbu@H)UPC)&zZPEtMv9#|~aK6ZJ;W zdoko8s6yCn5m^bvhi{+Tnwu=@S^$ddeG%`2%%9@)l2{!?<< zey&WCcD=nvPu90K$>at&*+kpLK65(!QPB*Y_L=FCT(N?ExHl+PV!=3b z{HM^9mb%xc?n<$m{7H=x8W?fe)NPv)tX{a+UDESSDvoIMygihBDa*U~@*`ib%rFlf z=ONYvn6|BdfXoGks~RGSng0b(w%P>6>H7tlacZE3z^`SVB!3$8K`oYgvj8yB2F_*< zrGf^zu*$W-fS@A;=$L5KXK!N|5~rKx{w*Vh$YN9AHT!Trt8vwNYsw3ZBkR3E%cf#= z8}bw?1atg2wotR;ZXCVaZX?f+nTqx40NNa*iQ`xawm4uH(hhiDIr;g|>%g>N7mHI> zOO34OtaLYM^E5w5x4*oB-We+i^mXHwdrD2)`u;TpQWF=<05;m_x&?iJK%Jz4HfN!FOo2sc|uCI&*(P(NaZf` z0g_U;KNU)bRDr$T9|Uu_{ol&_I?CburGm-Z_k^QRLo83fp6%~A z7%@3z9z-h%I_PN=scHEt%dSt8#+ErOHx7>*t_Cvs{_fNWD%a8}2$Ai;4(Ff64T0{f zU4EYS;_da60jl?G?u-U(nkdhD>GuPzwi~Id)`2QNWLqLmAYqBOC_VUV(&I$=>f)P| zucEy@C<4dgXXzF0_t8>h&gPrd<`(*Rf&KL242V57XR%{Pmt zDZ?sBl|~|eeE>*75qi2{Qsi|e2;;%Q$}m&)S1{4TlsCh0g4AB{>ZFNu18~s^#w_4H za^^bvq<2aVW$rfvf6oK9&BLn-hEk6RyA?%=nB$etK7;dX7o~CgWvA%IgFZm4G4HDS zYjEVKSx?Gbx|O5;PYcObAPhuew`sI`h}cK+2>715?b!_w?J6#%O&(O7GAFJ z)Ms@ib_=66yKlTY$WH}gH})inS|D{cnDG5}v-ZcriE%5waVcHa6E5}2Dx!0n_C^n< zY*ps?`UIIZeZ)bcRqwN_XZR;f0my=g_xeovbum>VormI zW2x1q=~0|Btxht!U;NTbO`FQ8$G*o1uG{bIT1EL226NvGl%Gp#D)Aa4U7aBO15Tg# zJ;Zvjc#Hqig7O$z%Co`ArnB@#x(R1wFqQlC=9&1Au~%o+-f&Jn^1W}FVJIxTIk{IO zjGp~H>`x7zJFDy_U1+-g(W&D$*V`)9sIN~Ky_eJQqGyVJ>GhBe8$ug-YwAIH7h~e& zAGM*A8*KilIBs~c^h8NUit*8jMx!kx> z`jq#ljIq;Yv|py^{UQ4&&Ekwb{tw}II~s)qO4Y;~oTnyZi2GdRpH2%@ZtO_?yXZ}? zZf)Mj5%n8w!&t`l;H=Z>?^-&Lx8wPxd5}K&%|?f-?`C46$!n%2K=YA&1t^Bzc(zkjwXJ%FMicb#@t)dOp4$jL5tuK%_z9GbeMYsA{dqOYB^lv$Z*MsHEM#a^ad+0fo{qPg|6pHgjh*w4>nY}%I9jl8=2 zypIr=a|$^lA>^<(cO*92UXr}V1`T6PWme0qQ5?Y5KOiAe-hMQP>n@Ze5m*$8?^RoB z+#7Q2sknt8#C*l0hKqR4wOvngg{Ge+1(|bFLsUU&w>+y-cqXr0h(&rwdLvqs>+h%uN*8c(I3yS512W$& z3@ovXP^it<2WX_=PH7Id69KZeoLn4j`3LZrHJS6o!;l@u}nB6aIKCe}vLJiFn@K@!AE{r*3iCPGgVNRg0!K*$Y42F#KpNC$|k=$c=Qs zapf?ivU{`TAw$eD{!F@4p1Acy5zr)70$d$0Nrp4=b5gMb4VC=0G^sj}@+@-1{FwTP zwFRAc(uR29Hf_2rsp;9>-s&XNjjv~&#M;punhV8rf6rGhPt7PIQJiibMAgILeFMUU zX1DyQN)_QMz<99K%YA8Edd3U0+_yKrXGTHQ`iYNrgLv7Pr49=}Yjb`jWs^`&FmDnR~mLXqURYlghv&`X4*13mGDR z=iQHN6G$)Ro^r+^k8}x+Psb{39Y!$A*1s=0d2t{@kKvK;jC1a|rYcDfCuVwUC?!a| z_y??Pu=qZhvS(@CLyXO^r_*Aiy&zJK*Pfdo|GF(aw6V6t4@9|F`K~Z1u|Qc*Q9z#v z29mGr^FZ%2YhY9(i_u=YsmAYBUDs~`YM(y%mn6&!d%HH0G>&ypIr%>;d^TT}s1Lv` zP}C|u%&t7PW&`O*R6Rm6KFC%VASYtP=TK2 zlIVLuu`?V6?S?*lWlk|M95Fv&2+-h}2QMZknshVQe;MCMHcR3GxAK&X*Rssuji}xB zIcIC(4jnBzeN@iN4ZCcPGEX?8JSy+LJs*>`ROH|N~zh*8QOkdexyT^013S2LH_$lLY z7;?Ahh?iw90xB~XrZyxj?4p=z_+~>$w~q&lA>M6ePLTxF z8u>?eerxO{7Ifo$1-cv~ff5XmhItyrpqD;peN8e83t-6=!6d6qHHUE-daKTYpe)8dgM)1yi~%qQ45;Eslx zpeegm%mIoZzz}<%IA6CS52*mwAjzwLSPnLSwI#z5Yf(Yf&`-1)h}^_;9;1EK6G|lv z1(~fThH7mm0!jt-EQ_(;(cbzvDimX1qi`&E+1y9J4~77O*DvTJJcxXvJNK*$>I0*O zz`+P5Rcj4XfN6Zs(YB3R3XDqD@d8~ySqeR-(Br8P@q)~I8(yNBE}mq-Ynh%LcwR=9 z8zpoxj0%fO9(g;tQ!Ec04C|j}B7z9N?*UwAoyIoT&F{ALKq5!36DKv?W*Y!(fl4C& zs|zqYN@-_OoXpgwDds#9)hjI$m6+upB+rEA{}Q>X`{(G8vSH8&O0 zH7V)PPs7t~MkM6|9BYM{E2{5)yot^a2*>heqN92rwDjUl{HqbkwRyV|xPUY9(~IP+ z+Te1l95)|DEsmEWPa6%VK1M9z1ucE=*vh^vF5Wq5Bv@`2NRpT7>1%ZT{csVnO7U!_ay*rq%5Ry;Jg11CRa10UhkME5Cz)c`Y2LKnKG9#L z(5Y1Qzp_IlW|zsF^P()Q836+9uDK@XKDp61SmHp#E}L^GG3^<)111Rh>j65Ewm+YE zzazOabG};NKxC9nFGs$GZsrPySy4M60XsHmdeLQgjLwX&}cd4wh z2|hwfBaE41yNP~OlqJh;mqG(`lG_$HzZ}cDRRQv*KL+$8GUY>}w>6D=STPZs8fYQX zuc|z^M`d$Q@o;eStS!&#)xJ^vQo*>dd=gMbFk5`m5*N$nZD`UrSnOQ47F!l_o3JiQ zcp*_QC@EQ|ay2gQu5a>|^mg^#cC-?%|JrcwT`$MLyQ5`C@puuu3k)w~kUzo3FARK) zfG)<6Q#nH*6Qe|wvIR`snwKUEXD|Q!VBPwbINvmpC#c7$S-;c}oNmrp zGcr47@1Z#ad70KPjmTdz12D88leCO}rHA00T3RHaEMs=AE79zb3=n3B?@)AD?;_%D z5YtMc--pB&Nlz?WE-w7=)~9I?L5mXhickkO7jCzRCz|~h!GIS1BZI86aUF=^{q5!R zz-iUg%1F;?5?Q|&M}@2hrTwzLX+09GeVm0mW8gfX8td>6U|*Aoe=i4r;~157E!P$| z85T+fxyiTmDC>B4nxGN)$3Qz}_kT0Zrak&=)Zxo8EhC?RMXHO8Ir}=DD4VI(dZ$6S zXjY10S86%(?!Ao5_eln-12y)fC#OH%9ebLDSSN5EWF!8 zdH?5Or}z+fpV=UQjvKhsAb?}cRI_N$1(#+CQrWs(GO~4Kl6WT7DI?Shn4sQOj9u>< z2CN<73cXQ`;_x!pJ!_%kSb)A$yva;vj94?wqG0j5#rZ{5;4xuD?0t@>V1--XPp@Mn z(YHkLVx3Aq0avo`DK;uAQENDn6d?0mX12K-IF}f?h^U5ys2&omnplZK-uD}@!vs7h zu3PWdT@He|CY>jK70;6n@zkjxq68pv%jO+DCWsL%R%(VQ_Of(^bFX|7vt)i!|AvKl zSosU5H;Z_73dNeUT#ughzO4V7h%BFk_;J%AFiz`$$J(D2ona-&cID$|82M`tTyueV z8mN4=LFhK>7z7qav(WM-#Mx#2W#+nq091xujk3`d0}ICDGG1;qJ!m~s+s@>j<}hV{KGxdYK& zV7CFf{$fiR=udyw?Xeyoi7lcQJWIJSn}`aEvt7V*sP}h?5{B%mcgyoR)0ERA-9o8w zn@+d>8E)Q-B6%#M(~+LwHpiJG)U(qe_wStaU6j}S;yq40Zevo$6GKiaWO{BiuI1FQ z%@ebAK1)@E-j8-y986_JUiKUfHZOt*Fy#Q&hCMFejRi=e@38uO_ z0<=yR>XzxH-MUEsPyR~E zK;z1DSHCBiX<7#=C0_4jwku{#M$6zO@Y2$%sNi*Da}20swn5r}=IEoOEtS^l|+T?42S@x1UB0v4mcv36_tFaeuD~!`9v8loqZxr22xNqv2(Tw+JW4v$2*w*D{&X= zfOin+NC|gPKUlXogDFFhP(oG%;UyFC#Z>UqjPiwI9a;8=;`bPTw(;C-N~1-$&qUss z6aV1$$6Dp4AhWq{&RThsvRC=j6Z2hO9TpxLTK8N9KN`Ttz;iHd5e$O-3mMYEf8}D zocoHrK3L=9{V2(FIOVqwW$|z2M-qtkWZh9SD5G*50LLib(Nm&&bMgbMGgbZgWxO)Q}ET2$e;It>XA*%xGTPh#0-t zhJ=d4!^N`}^3>;>0?WAK*2jyPJ}unHgS?kRV)J;E=ew|vkoXJz{ruI zEAqZ)-NB&klpiBGPHj4m2@9(i>keh5rAPAm$pkijv$)fHH|=J z(exbgL^o9M4-DuqZna(A)6$Y=9KcK&kiG$BnYNZHVZ{bvbfgCnT{L+duGFvf;iP$ZdB+O#6DSn>&+%M)7)~v~ zsB?F>c&mlkYiIGApl|7v+~7rBO4NS61G?t#bu7;Mx2I}^8yO%HBNKf&Iay|^*|w)M zy9;(eR{~EUn#JR6{FscJizQ27@qP*Ix#MGxHHuBB+wKycbE3zjv^6%otO-ow9D@2q zUiYTtV1*5Je9penaC8RDfU}lc(UrPiY5XoElE>4Dz8$7_K1JuH6>f2-N;s%sjA`Hy zHEIV84O}++4cZe^eXJRcvZ&)02!OJXFt}tGIb_@uG~56vf{Ub3+@=pFLl$*+RrBO% zcO%OKnGKT=li~JLYcdx&ITHaxR7kmp79Wv|TQcGIc{JJ-Kv_ariZj+o9+U^)jm4up zO2pV%{x1Rz52htC3;d#|R@Vt45u@PwnB zmCJdU@VCly@UY3^FB_BFGjMU)-BHtgqTPgw97cf4(Cyl&y&eqU-Y4=8;0Q;G6TRWy zV~l-OAWw=iydM-dWQpK2GNQB}}Ott=pdB1Ng6goke(#*d!9e`z4%RUweV# zZBec@oK=+M*O`aOpv5IaU7KZQe*-!dM{`uE^@Jf7^odS!OV%zLzzk7xiZWYX0W6|b zcS-P|!65Si6J`eiDGwDi80l@3oTa?q4`pUSw*_Lv@^9PcNI~(PA}s=u)eKPF+#=1C z`$X0}%kuXqa2MGyyF7BHX!!ps;mpIK{Qm!ckFjSrVlb!?qhuZXHWUUKgHYO$eH+S} zEtQOYjcH=UjHOW7L-7t{$=bHPzu5(_m^LoCXk5%g1 zXZ6n&^mQ8@7Y z-s=4*81QTjPp;?otcpQ6$+`mW*>U6`)4)$* zRV!ATDfQH{qQ*=wV%c@RZ_cYO&8f3CsmtetFXP=}RvRNjCvR)N5^ z2N-sLPUrduEaVeN@z0ovbnDq%5I5!6O4t-J*(J7Vo%jzq!MZf15vLOIQ6$O zR<1hGA6Kv6lMitc`1V02aWVod@Q4_hebVSh$h;%&RTV#Bzx1rBeWM?^JH3-E?gdB| z>~40poe0DZ@Y`C;fR6ZPSXO(Pydzi2`;<%9ie*XFM#f;|r}Vw9nJOMn&Ttg-hwC%^ zfG8$L#KK_Uw90_2yyqE5gjty6NKgp*bh=?JkoI8?uvl^E>5;utJ{RrXEQN(`f@u|d zNSgfx57tfXz2e9-H)lCo^-rv%WPpAQ(^GH zVN#X8EndPM!! z;s)8k@Iz$~6Io>LOttawWt;*`a32dEcHv@%Q^UY!qap1HY-FXtYo-x24zZ#~eFL>+ z_B7-x*^DKLz|1{sLFQh+o&)#0OL=CU?;i+{Hs7AH;t5_1I#ak+nb@>B_xB1!HbyN| z4MuF9`SrTY^Y{~q2AyQZJfjYFUch@HZIdQttsh9S$1G3*GqWMCKPaf|MQT*p#kM)} z5UZ!2tH#M>sqER$l;Y7?;Z~UeYvl-aaDzhH0#RT(OTcOm!hlkpqJ%!WGv_|opcrTb zQz}iftgU%)d)@tOn)BGE=7T%Ze4Vh@zD?Qrr+ymdtW_KJ>#tQ-( z%xsh6MvKod>nsW@kwF~tTNKXwH^l>4a?|lF-|zE#JNw`P3mBt=`|1{>_#3$z465T9 z7d5=z4Hi%JrP>`gqTbM<2Xwu;XK{;Au^6g(JX2Ba0?QNR0l0xMlAk~Hy*TEhaCyDR zs)^h5M@=ktZtHN^NLab8fmw7t=*NJB{+$YE`Tt7mo4%DTdoN1_Uc$$(_57^5ztdL# z#NN50U#6BQXJ#RxXtB-<+zP9__0(i@l9}~p^PW4p{na52a zOoQ?M5qe4S?8|wl48Kca6Ad3Z_bVlm&O|mIm&mj8q?w;|2HfPhH3z0e&E@s^ZTq`l zIueFmMWj-mk1smD86VS&diE{I>{~l%b+6KRzNCKat0Ri@gZdqjh1}4rgUD-j@ixPz z{W+i75}$@$E%;Cufur8yD=qEw2rQwaJ~*8T*!*oA-5|HE!PRy~8VyPFn<*|kD0LKC zmJ@_kTty08L9SSDQCmBSOA#UD${*-v1}5})Ot7=p=B#*U=lF^!yTVgB)pYFcfREQb zWj|Q`BZY-=-;ceB2GID}w)tn|A+>HK_Z98f(zOSV@X>)$yo94wr{kgNUyP2jCqCs4 ze&oO4RDENq{w=Y3%*CWTs4~^O)v@)HVznbQf);#?vW4X5o=C}~fK03a!8-`-A_!P8}@1r(eBa`F+L)ee~t4KHlI`nz%N zFPhQ*=TC<7CPsNE2*=H*Swt!av5qt&L9OJ%(_R5A`UXQlyS?;HiVAHZ$Zf_|; za+bEP@R8^w)>F>vm-!LH@-&Czvtc%nSZBsPyH|;jm$$23NYgReJ>3tTU&QRa81n*q zg`O3t|83R#2h^gV)Ug*2T{!ezNv|koU*@tV@UWLj-O|SPz7i&>D!~}8WU1-l3T?Vv z9#UR^I1Y;dU7Otl3PEbX%7#^3L3lAaIRp(TVdkF=2V1HSc=6+EAtD_X1mTWp7~%ez zoXwqofQyfoE_$^b#(E~I z`j!HfAz(RqlF z`PWEP5TRmO(Cc-1hC_`)mubKU?hisB&MTP*yw0~JL#Cd*UbC?oq0O+lwL z^}REow@0NF)Rs)+S;W=!n8yHfxeT(1Hkefd=}{9zSIaUB*SKZxmq#fr(D|&hi*&wMkY1d7iYAhbURV>*6jiBvzDB!QK^~$n2RVnOb zvg8nyr6LnEHzbw`%!N`GrdkH+Z{Ic04ssc1u6!o;UTWz)zup_p9&4Ztn3>D`!j~4> z_&Pm3_}mf7OIzA>#%)bZoE~h#sWNI?tTm}+R=1vi?R6>muCk4-yl>b@;hTIDsN&fl zI0ju2XMZo1SbjCBk@mR7E(p7~;;3N|>d?3)OVZ3dnR?~Uj@W?VKj0OGuLH~kmMOva z`E`~^JC`-S!L2ICmMVEG$&A}=uIZ*%HvOikgzr(-0UORP#W(R^@t6gcSiN2A{JDZw zuXnyK=Q#6OiwhMq$WdpqlRAaGt|OaXeHNp(RQ^rUG`vCcu6i&uaqry7g8Ajoo{;u# zC<1hT$SZmu*O6ysj>jKd1nZ~rVnvTu-JqRFJ_RFT49oqF+@&G=yy-pU7|C$o9(OWU zG^Is|YrllsD6wXb#z$uSPu_8?VfxjM=C2LMtFODRIKpG$a8~Ap);BTS$@O7_AZDnZ zKys}&Qyt66H(utL!FMkg>!2Y5F2@|guTEzS7@WS=Yo%$elJRIP0xbW3WN4kt-%eP@~@dzF~i{^LsQG_1T z>6QPV zv$deajHpT+lKZ8CJ#%(xd3XR|eUW&_on}NUOmcuw>uO>uLFXeMz_O4R(?6z#FeBvu`Hj0%SwNj#?>{@ zmg`9^yl2VD-IcH773}!0?*#-dnzPz*X0x9ziXwn}eA(RFB+$6s$kJ2NLz1XAdE^o+ zc_lSrn`*y33nRNGGp%A9<-y7UACk;`GD@*f!%n8_#o4oiCIdPNSL;!EG85!M+4t}IyoDLCn068;eLO0t*tyhm%-a1}xL)ctz1h?{>h~8p z309|Vc!xb+RfW!gZBa!rA-%_{yWdgp92fL2mp-)78hm7~z5P|E;`2Kh(af{CPvg-K zVi^MOou^1+qfV2uf=z?6m4ScNQ}v_;J)da)aN${f33P>1_!@9bQ=-r>lB2Y$e~|ZI zg|NFkVXQ0$OX_PFUf*h4Gt4r-;Ye1@-SQ=7ZKAjG%P+Hn&)%4WtykQ=Pw1qpO0Kz6 zhga|NbBR&-s*?*X)#J0s{NA%p^wFKP-6-8<^HojpM@E7NxqpLyMxA(`^lP9NG7c9A zx&+z4+eG?4w!+2oz1Il3@#US|h&=z@NZuCnlACjXbMA#HO_}qChU%vbKlD8dScHDB z)EzZ-$m;hE6ec}ycw1k~)o_m0ZL3A_O8uBz;Z2S#|9nJu&x=oN&;}+966ZJ5cF8(C zXuCeR&Vu`l_pz|bCxR&W&z|zjfe{R;VL%G2{})|IWwpbz97hbO<8(IC+L#0qXy{pl z@LPg`;_=VFZY2Gg@GJ4cTF>hupppcKs zi6Y4BEM;vXgUj!@8Fzh_l$BWD`4XY`=o|=b3ZL3|hhOE2Gh*DVN0!eegPCqj*U&eV zdA&K9s&V(3l3Pk)&>>-=p{O(Dty2Q}kNh7yQG++^Y&J2)W6QzjXFuB3J`Qf|;zCXg zv4uO~wxo#u{*@Qo@NW`+wYj`9lyE*LbbYd2>I*#CD{Jhn-lQR?uYZ4uo^Op%gIL@ybnLj?=8t$|R!y7ST$D72ODPz(lGM#p#) z?XXaULU2KC51(kDHfYrp%!;E|nk^Xc!ve(6`8+6=<1B{37rMJFelIcBKo)@~kk$+u zZA6-UG5q>& zUui(E;#q_nhl#UdG)O$7*l@GnkPbjQt`;Vbv))eEk_#!jGGbQV<2)1WCZT9R28% z=^oXUDA}z}l{qcY2apmQzjDD{N9K-P4$gI9kM^EDR-{2WBH-Q@$-1oo`+Mo-H%lPB zS4cBlj7^gKIrX51{&|c#m#ZtO)*^_gTIBB^&?W9Fi+Xe5Hmi#7W+G19RNY{_d57>M zEWH!CZ-I0q&4nlbG6%pGi0`$C$Qzrx=kQG#?ln6PLXW?+c`9Jk*&lf-XH{#|b`sNG zoN-P;q_E60L_Ay6@a`M9TvA_X7nk%O?;n4qCaiBqRfKtm>eL1ec~mhxp+ojM`ULA9 zsj1<+_wIZ3se5gW>Rl1qo{zkf$SgBW*saezXF(%+(i-Bxbb{Do!0b zi0v5Kad^ih7o82JHA3YAOA0Woj4<2d^eZE^qXTp4Jz0x5PiPef;zis+KBwwZ@NCJ! zt0E7L^-M5${`m1BZuYzvwr%hd8TSR4#xUeXu>F`1cGjS99Icz z!Mp5-54EDzF?{}wW%-9CUVN+bdD!Va3LB_L#>Q=hWhp|K5RId%i*Ph-$6yf?FmFI^^9Qum+HsFU>E6 z(B#1mhdv`9E{+kay8x;{SJ`DD`0&{nMQn*x(1%;yp!uBnjBI1vo{xUmX!9 zd&CW-5UVc;*~l~Uk#2iXV(@~j5O$X)9GA?;7Y2hot6usCj2lX~GXL&a3*uXQ*39w) z6>BGT2e8ksc9gk7*V9#k4;_ZY%}(=;Kr_QCBB;&h>-SYV7pXjua`g5)E3P-}(Q6Kc z4$M0%&DPG}-4gwgg6l@~D?4S|`CPxI)10SU&apLn+Kx|@o%ISSA&gkzIUJ;kwFx(0 zgyjve%))AxkCR^IcEOL|+fsx6yk(K`=c*wW~`rqQ2}%3zGNG>2RatZ3yw zM*gzZ)8wJ;~c4$oolfH!553_tvvF-L!8|w4iH<8_7o6_`O!jbiD*5&iBq7) ziEY?|qQ$)StU)=yHXS|MqXVNzf&hk7(eD!~HA&&?+9mGXC|8HCMLiYj*P~Wi4>Y<{U7(U+ku`C(k3%50-rFJ2zw}$OJ0%>K?GO(dy za*+i;@fXIkE*lZSGu;;58t|xM`S*3GqpdyK zD;}o;u)5~Y8faXC0R308fri=n<0QabMrDbV{8}#OP_Mid1;Nb9WyF=LvS9`_-~*l@ z`2S$nMFRL)6?qMFUj6z#wG`-TsOSY1vQ!@rl{Af;=&=mBF;c#qEB zl}u}8j%M|7=#(l!V1PhKG9&(+Km?d^6?k_Q{sT_v*&MAF9*H=<(2e>1VD0~Qph>Kr zdNR`nwqBb+L)Ab)WK~Z;yayOEtepY_7oT~rE;H1BCVT4}dLAC(A1*rt>6KC_@u+qV z76=EJxNtE4hYxsxuDg#PN=JzRza`&9kPc;+CIGUsBtPBfl&^I1OtnC27)JXCsVG1W zQ^9NUt-yi>vZH*mGU>wyt^E^3>}!GK(sUcBNTb^rc~+w@Y&X8>98>^F+J>bUuCMn4 zu3?4_g&p?i^cCV+EOz)VC-B5LKkdlmm7tSMJACSO1iGdPac`TFd|I8A#hvTO-l2(t zp<1#YaYqhm@V+W*jl^M>haP5%6=Y4LZVJEaWPDB+CvBFzUm=cId_Ux;lyGVm$XM3a zh#88m1uE2<6Tg#!3~PlCZOL<>8dSCFu~%i@^^;JQW-z23*w^&>i}*@jJ(JkAg(!Qs zxwgyG+;^TX1qMq|N{reV2S*sHG<5n*((vUc-_Mqvid8L_u<`wDHIo!?^ytoZFET*E z*uLkbkwBMlm-yLUO~&dcEq@fW*0o1_j%fkGP{$k&B@0I2+Grpi;iV9X_XPG}!ky}; za9;2-Nc-D`Inta2~MLTZ5E3aMR z3I@B>wE|Tua;le_DkBe(bVrQBVPxDxTaaX^d!$*KL?q)`4)a%yPJ1p;K$Y6K9lTIY zK-&7zk}L+ENEwnPpvQYuUE0|hsKTe#3dh;==gq==SqoG#0*~k?ORkD6 z4MYxgb6B$6ir5FC8^-M`E0$ka?T-a_>gm1b_Xc$5iTH?31y4vV{>w2^QvXWw5NrMm zaYDeFmCchh9k6CrcQH930O#G6_Yc@oubq7fhKwR^V2$@9wB!ZWA7myMBprS-4Lhb< zITz6VKvW>>@9P*{Pm*Cw*!zbthZs+%b*1xKYn&2#T^!DiBt!E&)bodFFOL#V3J)gz z`tRw2jBKLrCsQH%sB!nBm(y&g{x)sk)aCdtMMj~XlL>n-jXVT)hPKkZE`vx6$xo&c zVwTLZLe*zjst+%Z+2zObV-KU~?8Gy#;xY>^`gMnlJ1D4i@hd-!)skxKIhh za~vxg=e+0Y%F-i7f+Lc7qJ0I-S^t>NT;oXca}5ymw0{~n*ZV0mGCsZ^hG*|exUHGj zkZY6SFRz9%-qj+Q)`P6RQ=Db8po?e=rMS03Im!O`<>nf&snx>#XiVE{3&}L zt78PYPVJ{|w@Y;TNj@-_AioLj{=GbysMzQI_S)AUu1_TIa*PsD&@$FFJBB|pa|Es< zOGHsvc{@)S4%=dQCREfrRVNnW!NnOJ+s-L;e|2y;gzcf*M0omryS@v~QN+wK0w!8L zpT~+z9(^LADC|vb+AAgt;I{$E;dsaD`&_#YHF@zQ-lcZsl63qSZR zfJS*8um{B*z5*`&S;yYi))df>8Z*%YpGQv9c+W#j2;FE~48+BWdb(_YFBqJbik+=f z2Y(Y1&K8kUi2$EqU3m4JhyqIF5#ag)Zoy_-Uxas`69Mbav`Nag;UYK`5VT`6Xa@(j rOe#_UB1c5A7^(P8Gc6`cz!!u_87xncI6yt^PbioMflJ2x`}+R?rvfhH diff --git a/index.md b/index.md index 615dda6..a3adf86 100644 --- a/index.md +++ b/index.md @@ -1,26 +1,18 @@ --- -layout: lesson -root: . +site: sandpaper::sandpaper_site --- -This workshop is an introduction to parallel programming in Chapel. This -material is designed for Day 2 of HPC Carpentry. + + -By the end of this workshop, students will know: - -* the basic syntax of Chapel codes, -* how to run single-locale Chapel codes, -* how to write task-parallel codes for a shared-memory compute node, -* how to run multi-locale Chapel codes, -* how to write domain-parallel codes for a distributed-memory cluster. +This workshop is an introduction to parallel programming in Chapel. This material is designed for Day 2 of HPC Carpentry. -**NOTE: This is the draft HPC Carpentry release. Comments and feedback are -welcome.** +By the end of this workshop, students will know: -> ## Prerequisites -> -> There are no real prerequisites for this lesson, but prior programming and/or -> command line experience will be helpful. This lesson assumes no previous -> knowledge of parallel programming. -{: .prereq} +- the basic syntax of Chapel codes, +- how to run single-locale Chapel codes, +- how to write task-parallel codes for a shared-memory compute node, +- how to run multi-locale Chapel codes, +- how to write domain-parallel codes for a distributed-memory cluster. +**NOTE**: This is the draft HPC Carpentry release. Comments and feedback are welcome. diff --git a/instructors/instructor-notes.md b/instructors/instructor-notes.md new file mode 100644 index 0000000..d9a67aa --- /dev/null +++ b/instructors/instructor-notes.md @@ -0,0 +1,5 @@ +--- +title: 'Instructor Notes' +--- + +This is a placeholder file. Please add content here. diff --git a/learners/reference.md b/learners/reference.md new file mode 100644 index 0000000..ba26b9f --- /dev/null +++ b/learners/reference.md @@ -0,0 +1,8 @@ +--- +title: 'Reference' +--- + +## Glossary + +This is a placeholder file. Please add content here. + diff --git a/learners/setup.md b/learners/setup.md new file mode 100644 index 0000000..3bebcb4 --- /dev/null +++ b/learners/setup.md @@ -0,0 +1,55 @@ +--- +title: Setup +--- + +All hands-on work will likely be done on an HPC cluster with Chapel installed. Alternatively, you can run +Chapel via Docker on your computer. + + + + + + + + + + +## Software Setup + +::::::::::::::::::::::::::::::::::::::: discussion + +### Details + +To be added. + + + + + + +::::::::::::::::::::::::::::::::::::::::::::::::::: + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lesson-outline.md b/lesson-outline.md deleted file mode 100644 index 5fe4cad..0000000 --- a/lesson-outline.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -layout: page -title: HPC Carpentry Lesson Outline ---- - -# Lesson outline and todo list - -This is the tentative list of tasks and topics for each lesson. Lesson writers -are indicated with first/last initials (e.g. AR). Feel free to edit the topics -for your section. - -## 1. UNIX fundamentals - AR - -* SSH to a cluster -* Bash fundamentals (`cd`, `ls`, ..., aliases, functions, ~/.bashrc) -* Transferring files (`scp`? `sftp`? Maybe only one?) -* Overview of HPC resources - - * What is a cloud? - * What is a cluster? Different cluster types - * Overview of services available (Compute Canada, Amazon EC2, etc.) - -## 2. Submitting / running jobs - JS - -* Scheduler - lesson will cover SLURM (which can also run PBS scripts/commands - natively) - - * Submitting jobs - * Checking status of jobs - * Deleting jobs - * Job size consequences - * GUI vs. batch programs (X-forwarding, SSH tunnels?) - -* Using software and environment modules -* Playing friendly in the cluster - - * Understanding resource utilization - * Profiling code - time, size, etc. - * Getting system stats - * Consequences of going over - -## 3. Language refresher / introduction (Python - JB, Chapel - JZ+AR) - -* Programming language concepts - - * Compiled vs. interpreted languages - * How does a program work? - * Quick intro of programming language of choice - - * Major features + pros/cons - * What is it good at? - -* Actual language overview - - * Basic syntax (arithmetic, variables, etc.) - * Basic data structures (lists, arrays, etc.) - * Defining functions - * Conditional expressions - * For-loops - * Reading/writing data - -Some side notes: perhaps a quick refresh of key concepts right before use in -parallel section, certain concepts could get mixed in right before they're -needed by the parallel lesson. - -## 4. Intro to parallel programming (Python - JB, Chapel - JZ+AR) - -* Pipelining / automatic job submission / serial farming -* Shared memory programming -* Distributed memory programming -* Overview of good parallel design - - * Dependencies within own code - * Race conditions - -* Typical problems and bottlenecks - - * running in parallel (parallel scaling) - * parallel I/O (don't write a 1GB file from one processor if data is - already distributed, etc.) - * Storage limitations (millions of files, compression, text vs. binary - storage) - * Filesystem choice (home, scratch, tmp, etc.) - - -Good luck! diff --git a/links.md b/links.md new file mode 100644 index 0000000..4c5cd2f --- /dev/null +++ b/links.md @@ -0,0 +1,10 @@ + + +[pandoc]: https://pandoc.org/MANUAL.html +[r-markdown]: https://rmarkdown.rstudio.com/ +[rstudio]: https://www.rstudio.com/ +[carpentries-workbench]: https://carpentries.github.io/sandpaper-docs/ + diff --git a/profiles/learner-profiles.md b/profiles/learner-profiles.md new file mode 100644 index 0000000..434e335 --- /dev/null +++ b/profiles/learner-profiles.md @@ -0,0 +1,5 @@ +--- +title: FIXME +--- + +This is a placeholder file. Please add content here. diff --git a/reference.md b/reference.md deleted file mode 100644 index a9e9794..0000000 --- a/reference.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -layout: reference -permalink: /reference/ ---- - -## Glossary - diff --git a/setup.md b/setup.md deleted file mode 100644 index 692b779..0000000 --- a/setup.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -layout: page -title: Setup -root: . ---- - -There are several pieces of software you will wish to install before the -workshop. Though installation help will be provided at the workshop, we -recommend that these tools are installed (or at least downloaded) beforehand. - -> ## Bash and SSH -> -> This lesson requires a terminal application (`bash`, `zsh`, or others) with the -> ability to securely connect to a remote machine (`ssh`). -{: .prereq} - -## Where to type commands: How to open a new shell - -The shell is a program that enables us to send commands to the computer and -receive output. It is also referred to as the terminal or command line. - -Some computers include a default Unix Shell program. The steps below describe -some methods for identifying and opening a Unix Shell program if you already -have one installed. There are also options for identifying and downloading a -Unix Shell program, a Linux/UNIX emulator, or a program to access a Unix Shell -on a server. - -### Windows - -Computers with Windows operating systems do not automatically have a Unix Shell -program installed. In this lesson, we encourage you to use an emulator included -in Git for Windows, which gives you access to both Bash shell commands and Git. -If you have attended a Software Carpentry workshop session, it is likely you -have already received instructions on how to install Git for Windows. - -Once installed, you can open a terminal by running the program Git Bash from -the Windows start menu. - -#### Reference - -* [Git for Windows](https://gitforwindows.org/) — *Recommended* -* [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) - — advanced option for Windows 10 - -> ## Alternatives to Git for Windows -> -> Other solutions are available for running Bash commands on Windows. There is -> now a Bash shell command-line tool available for Windows 10. Additionally, -> you can run Bash commands on a remote computer or server that already has a -> Unix Shell, from your Windows machine. This can usually be done through a -> Secure Shell (SSH) client. One such client available for free for Windows -> computers is PuTTY. See the reference below for information on installing and -> using PuTTY, using the Windows 10 command-line tool, or installing and using -> a Unix/Linux emulator. -> -> For advanced users, you may choose one of the following alternatives: -> -> * Install the [Windows Subsystem for -> Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) -> * Use the Windows [Powershell]( -https://docs.microsoft.com/en-us/powershell/scripting/learn/remoting/ssh-remoting-in-powershell-core?view=powershell-7) -> * Read up on [Using a Unix/Linux emulator (Cygwin) or Secure Shell (SSH) -> client (Putty)](http://faculty.smu.edu/reynolds/unixtut/windows.html) -> -> > ## Warning -> > -> > Commands in the Windows Subsystem for Linux (WSL), Powershell, or Cygwin -> > may differ slightly from those shown in the lesson or presented in the -> > workshop. Please ask if you encounter such a mismatch — you're -> > robably not alone. -> {: .challenge} -{: .discussion} - -### macOS - -On macOS, the default Unix Shell is accessible by running the Terminal program -from the `/Application/Utilities` folder in Finder. - -To open Terminal, try one or both of the following: - -* In Finder, select the Go menu, then select Utilities. Locate Terminal in the - Utilities folder and open it. -* Use the Mac ‘Spotlight’ computer search function. Search for: `Terminal` and - press Return. - -#### Reference - -[How to Use Terminal on a -Mac](http://www.macworld.co.uk/feature/mac-software/how-use-terminal-on-mac-3608274/) - -### Linux - -On most versions of Linux, the default Unix Shell is accessible by running the -[(Gnome) Terminal](https://help.gnome.org/users/gnome-terminal/stable/) or -[(KDE) Konsole](https://konsole.kde.org/) or -[xterm](https://en.wikipedia.org/wiki/Xterm), which can be found via the -applications menu or the search bar. - -### Special cases - -If none of the options above address your circumstances, try an online search -for: `Unix shell [your operating system]`. - -## SSH for Secure Connections - -All students should have an SSH client installed. SSH is a tool that allows us -to connect to and use a remote computer as our own. - -### Windows - -Git for Windows comes with SSH preinstalled: you do not have to do anything. - -> ## GUI Support -> -> If you know that the software you will be running on the cluster requires a -> graphical user interface (a GUI window needs to open for the application to -> run properly), please install [MobaXterm](http://mobaxterm.mobatek.net) Home -> Edition. -{: .discussion} - -### macOS - -macOS comes with SSH pre-installed: you do not have to do anything. - -> ## GUI Support -> -> If you know that the software you will be running requires a graphical user -> interface, please install [XQuartz](www.xquartz.org). -{: .discussion} - -### Linux - -Linux comes with SSH and X window support preinstalled: you do not have to do -anything. diff --git a/site/README.md b/site/README.md new file mode 100644 index 0000000..42997e3 --- /dev/null +++ b/site/README.md @@ -0,0 +1,2 @@ +This directory contains rendered lesson materials. Please do not edit files +here. From 998b54cd139dbbd654f77a392f9d20601a208c44 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 13 Sep 2024 14:34:26 -0700 Subject: [PATCH 02/70] update CITATION and CITATION.cff --- CITATION | 4 +++- CITATION.cff | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CITATION b/CITATION index d5bfc2b..23077f9 100644 --- a/CITATION +++ b/CITATION @@ -1 +1,3 @@ -FIXME: describe how to cite this lesson. +To reference this lesson, please cite: + +Razoumov A., Zuniga J. (2024). Introduction to High-Performance Computing in Chapel. https://www.hpc-carpentry.org/hpc-chapel diff --git a/CITATION.cff b/CITATION.cff index a2c8a66..b6d4dfa 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,7 +5,7 @@ # ensuring that the author list and other fields remain accurate. cff-version: 1.2.0 -title: FIXME +title: Introduction to High-Performance Computing in Chapel message: >- Please cite this lesson using the information in this file when you refer to it in publications, and/or if you @@ -13,10 +13,17 @@ message: >- training material. type: dataset authors: - - given-names: FIXME - family-names: FIXME + - given-names: Alex + family-names: Razoumov + email: alex.razoumov@westdri.ca + affiliation: SFU + - given-names: Juan + family-names: Zuniga +repository-code: 'https://github.com/hpc-carpentry/hpc-chapel' +url: 'https://www.hpc-carpentry.org/hpc-chapel' abstract: >- - FIXME Replace this with a short abstract describing the - lesson, e.g. its target audience and main intended - learning objectives. + This lesson is an introduction to high-performance + computing using Chapel parallel language. +keywords: + - 'Chapel, HPC, parallel' license: CC-BY-4.0 From 546ea9c4aa580f161a5bfc8b65c9e00bd3abc732 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 08:59:44 -0700 Subject: [PATCH 03/70] rename FIXME.Rproj to hpc-chapel.Rproj --- FIXME.Rproj => hpc-chapel.Rproj | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename FIXME.Rproj => hpc-chapel.Rproj (100%) diff --git a/FIXME.Rproj b/hpc-chapel.Rproj similarity index 100% rename from FIXME.Rproj rename to hpc-chapel.Rproj From 88ac5286fbee2cd1898a0b223bef27e8f57fe364 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 09:02:08 -0700 Subject: [PATCH 04/70] edit FIXME tags in CONTRIBUTING.md and config.yaml --- CONTRIBUTING.md | 6 +++--- config.yaml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c2b81c..30e6179 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,9 +102,9 @@ community listed at including via social media, slack, newsletters, and email lists. You can also [reach us by email][contact]. -[repo]: https://example.com/FIXME -[repo-issues]: https://example.com/FIXME/issues -[contact]: mailto:team@carpentries.org +[repo]: https://github.com/hpc-carpentry/hpc-chapel +[repo-issues]: https://github.com/hpc-carpentry/hpc-chapel/issues +[contact]: mailto:maintainers-hpc@lists.carpentries.org [cp-site]: https://carpentries.org/ [dc-issues]: https://github.com/issues?q=user%3Adatacarpentry [dc-lessons]: https://datacarpentry.org/lessons/ diff --git a/config.yaml b/config.yaml index 0d1751a..395556b 100644 --- a/config.yaml +++ b/config.yaml @@ -21,26 +21,26 @@ carpentry_description: 'Introduction to parallel programming in Chapel' title: 'Introduction to High-Performance Computing in Chapel' # Date the lesson was created (YYYY-MM-DD, this is empty by default) -created: ~ # FIXME +created: 2017-09-14 # Comma-separated list of keywords for the lesson -keywords: 'software, data, lesson, The Carpentries, HPC, Chapel' # FIXME +keywords: 'software, data, lesson, The Carpentries, HPC, Chapel' # Life cycle stage of the lesson # possible values: pre-alpha, alpha, beta, stable -life_cycle: 'alpha' # FIXME +life_cycle: 'alpha' # License of the lesson license: 'CC-BY 4.0' # Link to the source repository for this lesson -source: 'https://github.com/carpentries/workbench-template-md' # FIXME +source: 'https://github.com/hpc-carpentry/hpc-chapel' # Default branch of your lesson branch: 'main' # Who to contact if there are any issues -contact: 'team@carpentries.org' # FIXME +contact: 'maintainers-hpc@lists.carpentries.org' # Navigation ------------------------------------------------ # From a345b2e468f58d363c02c45a3ef0999ef25c6462 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 09:24:57 -0700 Subject: [PATCH 05/70] change all Carpentries references and links to High Performance Computing Carpentry --- LICENSE.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 7632871..2e20476 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -4,9 +4,9 @@ title: "Licenses" ## Instructional Material -All Carpentries (Software Carpentry, Data Carpentry, and Library Carpentry) -instructional material is made available under the [Creative Commons -Attribution license][cc-by-human]. The following is a human-readable summary of +All High Performance Computing Carpentry instructional material is +made available under the [Creative Commons Attribution +license][cc-by-human]. The following is a human-readable summary of (and not a substitute for) the [full legal text of the CC BY 4.0 license][cc-by-legal]. @@ -23,8 +23,8 @@ terms. Under the following terms: - **Attribution**---You must give appropriate credit (mentioning that your work - is derived from work that is Copyright (c) The Carpentries and, where - practical, linking to ), provide a [link to the + is derived from work that is Copyright (c) High Performance Computing Carpentry and, where + practical, linking to ), provide a [link to the license][cc-by-human], and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. @@ -45,7 +45,7 @@ Notices: ## Software Except where otherwise noted, the example programs and other software provided -by The Carpentries are made available under the [OSI][osi]-approved [MIT +by High Performance Computing Carpentry are made available under the [OSI][osi]-approved [MIT license][mit-license]. Permission is hereby granted, free of charge, to any person obtaining a copy of From 63c3f859111fe51666c5d59860bf1524bdf9fcbe Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 09:30:49 -0700 Subject: [PATCH 06/70] remove references to Cray (that no longer exists); add open-source in the description --- README.md | 2 +- episodes/01-intro.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1945665..c965428 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ budgeted half a day's worth of teaching-time, resulting in a two day workshop. 4. Introduction to parallel programming Sections 3 and 4 (programming) will feature two programming languages: -[Python](https://www.python.org/) and [Chapel](http://chapel.cray.com/). There +[Python](https://www.python.org/) and [Chapel](https://chapel-lang.org). There are strong arguments for both languages, and instructors will be able to choose which language they wish to teach in. diff --git a/episodes/01-intro.md b/episodes/01-intro.md index 17237e9..e1890b2 100644 --- a/episodes/01-intro.md +++ b/episodes/01-intro.md @@ -12,7 +12,7 @@ exercises: 15 - "Write and execute our first chapel program." :::::::::::::::::::::::::::::::::::::::::::::::: -**_Chapel_** is a modern programming language, developed by _Cray Inc._, that supports HPC via high-level +**_Chapel_** is a modern, open-source programming language that supports HPC via high-level abstractions for data parallelism and task parallelism. These abstractions allow the users to express parallel codes in a natural, almost intuitive, manner. In contrast with other high-level parallel languages, however, Chapel was designed around a _multi-resolution_ philosophy. This means that users can incrementally add more From 93e6b622bea0f0ef2579db0ded00027c185971fc Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 09:36:54 -0700 Subject: [PATCH 07/70] fix the 'form' typo in the code and output --- episodes/13-synchronization.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index 161cf33..e625acf 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -224,7 +224,7 @@ lock.write(0); //the main task set lock to zero coforall id in 1..numtasks { - writeln("greetings form task ",id,"... I am waiting for all tasks to say hello"); + writeln("greetings from task ",id,"... I am waiting for all tasks to say hello"); lock.add(1); //task id says hello and atomically adds 1 to lock lock.waitFor(numtasks); //then it waits for lock to be equal numtasks (which will happen when all tasks say hello) writeln("task ",id," is done..."); @@ -237,11 +237,11 @@ coforall id in 1..numtasks ``` ```output -greetings form task 4... I am waiting for all tasks to say hello -greetings form task 5... I am waiting for all tasks to say hello -greetings form task 2... I am waiting for all tasks to say hello -greetings form task 3... I am waiting for all tasks to say hello -greetings form task 1... I am waiting for all tasks to say hello +greetings from task 4... I am waiting for all tasks to say hello +greetings from task 5... I am waiting for all tasks to say hello +greetings from task 2... I am waiting for all tasks to say hello +greetings from task 3... I am waiting for all tasks to say hello +greetings from task 1... I am waiting for all tasks to say hello task 1 is done... task 5 is done... task 2 is done... From fc3b50f9e7ab511c77a1998f9e16ec3752b4168c Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 10:13:50 -0700 Subject: [PATCH 08/70] rewrite text in 14-parallel-case-study.md to remove references to previously deleted Exercise 2 --- episodes/14-parallel-case-study.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/episodes/14-parallel-case-study.md b/episodes/14-parallel-case-study.md index af730aa..72fce50 100644 --- a/episodes/14-parallel-case-study.md +++ b/episodes/14-parallel-case-study.md @@ -12,10 +12,12 @@ exercises: 30 - "First objective." :::::::::::::::::::::::::::::::::::::::::::::::: -The parallelization of our base solution for the heat transfer equation can be achieved following the ideas of -Exercise 2. The entire grid of points can be divided and assigned to multiple tasks. Each tasks should compute -the new temperature of its assigned points, and then we must perform a **_reduction_**, over the whole grid, -to update the greatest difference in temperature. +Here is our plan to task-parallelize the heat transfer equation: + +1. divide the entire grid of points into blocks and assign blocks to individual tasks, +1. each task should compute the new temperature of its assigned points, +1. perform a **_reduction_** over the whole grid, to update the greatest temperature difference between `Tnew` + and `T`. For the reduction of the grid we can simply use the `max reduce` statement, which is already parallelized. Now, let's divide the grid into `rowtasks` x `coltasks` sub-grids, and assign each sub-grid to a @@ -45,9 +47,11 @@ while (c=mindif) do { } ``` -Note that now the nested for loops run from `rowi` to `rowf` and from `coli` to `colf` which are, +Note that now the nested `for` loops run from `rowi` to `rowf` and from `coli` to `colf` which are, respectively, the initial and final row and column of the sub-grid associated to the task `taskid`. To compute -these limits, based on `taskid`, we can again follow the same ideas as in Exercise 2. +these limits, based on `taskid`, we need to compute the number of rows and columns per task (`nr` and `nc`, +respectively) and account for possible non-zero remainders (`rr` and `rc`) that we should add to the last row +and column: ```chpl config const rowtasks = 2; From 7f36b657e45a5871b2a59ea90fe730fe2f7dfeb1 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 10:28:30 -0700 Subject: [PATCH 09/70] remove prompt characters from bash blocks in all files --- episodes/03-ranges-arrays.md | 4 ++-- episodes/05-loops.md | 20 ++++++++++---------- episodes/07-commandargs.md | 6 +++--- episodes/08-timing.md | 6 +++--- episodes/12-fire-forget-tasks.md | 16 ++++++++-------- episodes/13-synchronization.md | 12 ++++++------ episodes/14-parallel-case-study.md | 8 ++++---- episodes/21-locales.md | 18 +++++++++--------- 8 files changed, 45 insertions(+), 45 deletions(-) diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index f12585d..cf51e91 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -178,8 +178,8 @@ writeln('Temperature at start is: ', temp[x, y]); ``` ```bash ->> chpl base_solution.chpl -o base_solution ->> ./base_solution +chpl base_solution.chpl -o base_solution +./base_solution ``` ```output diff --git a/episodes/05-loops.md b/episodes/05-loops.md index f967234..9119618 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -79,8 +79,8 @@ statement. Now let's compile and execute our code again: ```bash ->> chpl base_solution.chpl -o base_solution ->> ./base_solution +chpl base_solution.chpl -o base_solution +./base_solution ``` ```output @@ -136,8 +136,8 @@ To see the evolution of the temperature at the top right corner of the plate, we `y`. This corner correspond to the first row (`x=1`) and the last column (`y=cols`) of the plate. ```bash ->> chpl base_solution.chpl -o base_solution ->> ./base_solution +chpl base_solution.chpl -o base_solution +./base_solution ``` ```output @@ -193,8 +193,8 @@ The division of integers in Chapel returns an integer, then, as `rows` and `cols 80 as real so that the quotient is not truncated. ```bash ->> chpl base_solution.chpl -o base_solution ->> ./base_solution +chpl base_solution.chpl -o base_solution +./base_solution ``` ```output @@ -247,8 +247,8 @@ Clearly there is no need to keep the difference at every single position in the `curdif` if we find a greater one. ```bash ->> chpl base_solution.chpl -o base_solution ->> ./base_solution +chpl base_solution.chpl -o base_solution +./base_solution ``` ```output @@ -283,8 +283,8 @@ writeln('The difference in temperatures between the last two iterations was: ',c and compile and execute our final code, ```bash ->> chpl base_solution.chpl -o base_solution ->> ./base_solution +chpl base_solution.chpl -o base_solution +./base_solution ``` ```output diff --git a/episodes/07-commandargs.md b/episodes/07-commandargs.md index 782eeec..a663a26 100644 --- a/episodes/07-commandargs.md +++ b/episodes/07-commandargs.md @@ -29,13 +29,13 @@ config const niter = 500; //number of iterations ``` ```bash ->> chpl base_solution.chpl -o base_solution +chpl base_solution.chpl -o base_solution ``` it can be initialised with a specific value, when executing the code at the command line, using the syntax: ```bash ->> ./base_solution --niter=3000 +./base_solution --niter=3000 ``` ```output @@ -71,7 +71,7 @@ for 10000 iterations or until the difference of temperature between iterations i print the temperature every 1000 iterations. ```bash ->> ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 ``` ```output diff --git a/episodes/08-timing.md b/episodes/08-timing.md index 1a37756..c5e51ce 100644 --- a/episodes/08-timing.md +++ b/episodes/08-timing.md @@ -20,7 +20,7 @@ But first, we need a quantitative way to measure the performance of our code. T see how long it takes to finish a simulation. The UNIX command `time` could be used to this effect ```bash ->> time ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +time ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 ``` ```output @@ -81,8 +81,8 @@ writeln('The greatest difference in temperatures between the last two iterations ``` ```bash ->> chpl base_solution.chpl -o base_solution ->> ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +chpl base_solution.chpl -o base_solution +./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 ``` ```output diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index df0c58c..7c683dc 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -45,8 +45,8 @@ writeln('this is main thread, I am done...'); ``` ```bash ->> chpl begin_example.chpl -o begin_example ->> ./begin_example +chpl begin_example.chpl -o begin_example +./begin_example ``` ```output @@ -157,8 +157,8 @@ writeln("this message won't appear until all tasks are done..."); ``` ```bash ->> chpl cobegin_example.chpl -o cobegin_example ->> ./cobegin_example +chpl cobegin_example.chpl -o cobegin_example +./cobegin_example ``` ```output @@ -199,8 +199,8 @@ writeln("this message won't appear until all tasks are done..."); ``` ```bash -> > chpl coforall_example.chpl -o coforall_example -> > ./coforall_example --numoftasks=5 +chpl coforall_example.chpl -o coforall_example +./coforall_example --numoftasks=5 ``` ```output @@ -326,8 +326,8 @@ for i in 0..numoftasks-1 do ``` ```bash -> > chpl --fast exercise_coforall_2.chpl -o exercise_coforall_2 -> > ./exercise_coforall_2 +chpl --fast exercise_coforall_2.chpl -o exercise_coforall_2 +./exercise_coforall_2 ``` ```output diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index e625acf..fc3f701 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -49,8 +49,8 @@ writeln('this is main thread, I am done...'); ``` ```bash ->> chpl sync_example_1.chpl -o sync_example_1 ->> ./sync_example_1 +chpl sync_example_1.chpl -o sync_example_1 +./sync_example_1 ``` ```output @@ -154,8 +154,8 @@ writeln("and now it is done"); ``` ```bash ->> chpl sync_example_2.chpl -o sync_example_2 ->> ./sync_example_2 +chpl sync_example_2.chpl -o sync_example_2 +./sync_example_2 ``` ```output @@ -232,8 +232,8 @@ coforall id in 1..numtasks ``` ```bash ->> chpl atomic_example.chpl -o atomic_example ->> ./atomic_example +chpl atomic_example.chpl -o atomic_example +./atomic_example ``` ```output diff --git a/episodes/14-parallel-case-study.md b/episodes/14-parallel-case-study.md index 72fce50..75e23f2 100644 --- a/episodes/14-parallel-case-study.md +++ b/episodes/14-parallel-case-study.md @@ -104,8 +104,8 @@ distribution for us. We will study data parallelism in the following lessons, bu benchmark solution with our `coforall` parallelization to see how the performance improved. ```bash -> > chpl --fast parallel_solution_1.chpl -o parallel1 -> > ./parallel1 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +chpl --fast parallel_solution_1.chpl -o parallel1 +./parallel1 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 ``` ```output @@ -290,8 +290,8 @@ coforall taskid in 0..coltasks*rowtasks-1 do Using the solution in the Exercise 4, we can now compare the performance with the benchmark solution ```bash ->> chpl --fast parallel_solution_2.chpl -o parallel2 ->> ./parallel2 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +chpl --fast parallel_solution_2.chpl -o parallel2 +./parallel2 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 ``` ```output diff --git a/episodes/21-locales.md b/episodes/21-locales.md index 0150154..b0b94d8 100644 --- a/episodes/21-locales.md +++ b/episodes/21-locales.md @@ -22,8 +22,8 @@ a multi-locale cluster, so you would compile and launch multi-locale Chapel code number of locales with `-nl` flag: ```bash -$ chpl --fast mycode.chpl -o mybinary -$ ./mybinary -nl 4 +chpl --fast mycode.chpl -o mybinary +./mybinary -nl 4 ``` Inside the Docker container on multiple locales your code will not run any faster than on a single locale, @@ -42,8 +42,8 @@ be always possible due to a number of factors), then you would simply compile th binary `mybinary` to submit the job to the queue: ```bash -$ chpl --fast mycode.chpl -o mybinary -$ ./mybinary -nl 2 +chpl --fast mycode.chpl -o mybinary +./mybinary -nl 2 ``` The exact parameters of the job such as the maximum runtime and the requested memory can be specified with @@ -58,8 +58,8 @@ the InfiniBand interconnect on Graham, Béluga, Narval) modules, so -- depending load Chapel using one of the two lines below: ```bash -$ module load gcc chapel-ofi # for the OmniPath interconnect on Cedar cluster -$ module load gcc chapel-ucx # for the InfiniBand interconnect on Graham, Béluga, Narval clusters +module load gcc chapel-ofi # for the OmniPath interconnect on Cedar cluster +module load gcc chapel-ucx # for the InfiniBand interconnect on Graham, Béluga, Narval clusters ``` @@ -68,9 +68,9 @@ We can also launch multi-locale Chapel codes using the real executable `mybinary interactive job you would type: ```bash -$ salloc --time=0:30:0 --nodes=4 --cpus-per-task=3 --mem-per-cpu=1000 --account=def-guest -$ chpl --fast mycode.chpl -o mybinary -$ srun ./mybinary_real -nl 4 # will run on four locales with max 3 cores per locale +salloc --time=0:30:0 --nodes=4 --cpus-per-task=3 --mem-per-cpu=1000 --account=def-guest +chpl --fast mycode.chpl -o mybinary +srun ./mybinary_real -nl 4 # will run on four locales with max 3 cores per locale ``` Production jobs would be launched with `sbatch` command and a Slurm launch script as usual. From ce5893dc8b77bae37a70a6b584057d29365941dc Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 10:49:23 -0700 Subject: [PATCH 10/70] make the keypoints in 14-parallel-case-study.md more specific --- episodes/14-parallel-case-study.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/episodes/14-parallel-case-study.md b/episodes/14-parallel-case-study.md index 75e23f2..a5e4f6c 100644 --- a/episodes/14-parallel-case-study.md +++ b/episodes/14-parallel-case-study.md @@ -342,5 +342,9 @@ coforall (i,j) in {1..n,1..n} by (stride,stride) { // 5x5 decomposition into 20x We will study data parallelism in more detail in the next section. ::::::::::::::::::::::::::::::::::::: keypoints -- "There are many ways to implement task parallelism for the diffusion solver." +- "To parallelize the diffusion solver with tasks, you divide the 2D domain into blocks and assign each block + to a task." +- "To get the maximum performance, you need to launch the parallel tasks only once, and run the temporal loop + of the simulation with the same set of tasks, resuming the main task only to print the final results." +- "Parallelizing with tasks is more laborious than parallelizing with data (covered in the next section)." :::::::::::::::::::::::::::::::::::::::::::::::: From 9545546cda4cf070e3499c7330e26cf3cdf65594 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 11:35:23 -0700 Subject: [PATCH 11/70] add Chapel installation instructions for Mac, Linux, Windows; Windows instructions are more of a placeholder for someone to test --- learners/setup.md | 62 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/learners/setup.md b/learners/setup.md index 3bebcb4..22f1dd9 100644 --- a/learners/setup.md +++ b/learners/setup.md @@ -2,8 +2,8 @@ title: Setup --- -All hands-on work will likely be done on an HPC cluster with Chapel installed. Alternatively, you can run -Chapel via Docker on your computer. +We highly recommend running Chapel on an HPC cluster. Alternatively, you can run Chapel on your computer, but +don't expect a multi-node speedup since you have only one node. @@ -20,7 +20,8 @@ Chapel via Docker on your computer. ### Details -To be added. +This section describes installing Chapel on your own computer. Before proceeding, please double-check that +your workshop instructors do not already provide Chapel on an HPC cluster. @@ -29,27 +30,58 @@ To be added. ::::::::::::::::::::::::::::::::::::::::::::::::::: - +:::::::::::::::: spoiler - +### Windows - +Go to the website https://docs.docker.com/docker-for-windows/install/ and download the Docker Desktop +installation file. Double-click on the `Docker_Desktop_Installer.exe` to run the installer. During the +installation process, enable Hyper-V Windows Feature on the Configuration page, and wait for the installation +to complete. At this point you might need to restart your computer. - +Eventually you want to run https://hub.docker.com/r/chapel/chapel Docker image. - +:::::::::::::::::::::::: - +:::::::::::::::: spoiler - +### MacOS - +The quickest way to get started with Chapel on MacOS is to install it via Homebrew. If you don't have Homebrew +installed (skip this step if you do), open Terminal.app and type +```bash +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +``` - +Next, proceed to installing Chapel: - +```bash +brew update +brew install chapel +``` - + - + + + + + +:::::::::::::::::::::::: + + +:::::::::::::::: spoiler + +### Linux + +At https://github.com/chapel-lang/chapel/releases scroll to the first "Assets" section (you might need to +click on "Show all assets") and pick the latest precompiled Chapel package for your Linux distribution. For +example, with Ubuntu 22.04 you can do: + +```bash +wget https://github.com/chapel-lang/chapel/releases/download/2.0.0/chapel-2.1.0-1.ubuntu22.amd64.deb +sudo apt install ./chapel-2.1.0-1.ubuntu22.amd64.deb +``` + +:::::::::::::::::::::::: From 9b606c0d51a918119d5df02b675527aab085390b Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 12:32:01 -0700 Subject: [PATCH 12/70] add more generic 'module avail chapel' advice instead of the specific module suggestion; remove .checklist --- episodes/01-intro.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/episodes/01-intro.md b/episodes/01-intro.md index e1890b2..6e5e37a 100644 --- a/episodes/01-intro.md +++ b/episodes/01-intro.md @@ -62,15 +62,14 @@ If we can see this, everything works! Depending on the code, it might utilise several or even all cores on the current node. The command above implies that you are allowed to utilise all cores. This might not be the case on an HPC cluster, where a login node is shared by many people at the same time, and where it might not be a good idea to occupy all cores on a -login node with CPU-intensive tasks. +login node with CPU-intensive tasks. Instead, you will need to submit your Chapel run as a job to the +scheduler asking for a specific number of CPU cores. -On Compute Canada clusters Cedar and Graham we have two versions of Chapel, one is a single-locale -(single-node) Chapel, and the other is a multi-locale (multi-node) Chapel. For now, we will start with -single-locale Chapel. If you are logged into Cedar or Graham, you'll need to load the single-locale Chapel -module: +Use `module avail chapel` to list Chapel packages on your HPC cluster, and select the best fit for Chapel, +e.g. the single-locale Chapel module: ```bash -module load gcc chapel-multicore +module load chapel-multicore ``` Then, for running a test code on a cluster you would submit an interactive job to the queue @@ -134,7 +133,6 @@ So, our objective is to: > the cluster > 3. Use data parallelism to improve the performance of the code and run it in > the cluster. -{: .checklist} ::::::::::::::::::::::::::::::::::::: keypoints - "Chapel is a compiled language - any programs we make must be compiled with `chpl`." From a5543b60063dc5c26db464c682e0e5cb9cb00498 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 16 Sep 2024 12:53:49 -0700 Subject: [PATCH 13/70] corrected a typo --- episodes/01-intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/01-intro.md b/episodes/01-intro.md index 6e5e37a..d401897 100644 --- a/episodes/01-intro.md +++ b/episodes/01-intro.md @@ -110,7 +110,7 @@ points, and to evaluate the temperature on each point at each iteration, accordi **_difference equation_**: ```chpl -T[i,j] = 0.25 (Tp[i-1,j] + Tp[i+1,j] + Tp[i,j-1] + Tp[i,j+1]) +T[i,j] = 0.25 * (Tp[i-1,j] + Tp[i+1,j] + Tp[i,j-1] + Tp[i,j+1]) ``` Here `T` stands for the temperature at the current iteration, while `Tp` contains the temperature calculated From 5c72240268394bb536e0da8cffdc7d167ac3a008 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 16:08:50 -0700 Subject: [PATCH 14/70] rewrite wording around 'basic maths' --- episodes/02-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index a136b1a..f515edc 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -15,7 +15,7 @@ exercises: 15 - "Know how to define and use data stored as variables." :::::::::::::::::::::::::::::::::::::::::::::::: -Basic maths in Chapel works the same as other programming languages. Try compiling the following code to see +Using basic maths in Chapel is fairly intuitive. Try compiling the following code to see how the different mathematical operators work. ```chpl From 6cd1e508a96856cb404b6f21665e57c72bcdd357 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 16:20:48 -0700 Subject: [PATCH 15/70] remove the paragraph distinguishing variable initialization and variable assignment --- episodes/02-variables.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index f515edc..7626048 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -66,10 +66,6 @@ data while executing a program. A variable has three elements: 2. a **_type_**, that indicates the kind of data that we can store in it, and 3. a **_value_**, the actual information or data stored in the variable. -When we store a value in a variable for the first time, we say that we **_initialised_** it. Further changes -to the value of a variable are called **_assignments_**, in general, `x=a` means that we assign the value *a* -to the variable *x*. - Variables in Chapel are declared with the `var` or `const` keywords. When a variable declared as const is initialised, its value cannot be modified anymore during the execution of the program. From 573402db234b34538d24757c9d6de433b1b339e5 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 16:57:13 -0700 Subject: [PATCH 16/70] add a call-out where we try to use an un-initialized variable --- episodes/02-variables.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index 7626048..abff4f0 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -69,7 +69,28 @@ data while executing a program. A variable has three elements: Variables in Chapel are declared with the `var` or `const` keywords. When a variable declared as const is initialised, its value cannot be modified anymore during the execution of the program. -In Chapel, to declare a variable we must specify the type of the variable, or initialise it in place with some + + +::::::::::::::::::::::::::::::::::::: callout + +In the following code (saved as `variables.chpl`) we have not initialised the variable `test` before trying to +use it in line 2: + +```chpl +const test; // declare 'test' variable +writeln('The value of test is: ', test); +``` +```error +variables.chpl:1: error: 'test' is not initialized and has no type +variables.chpl:1: note: cannot find initialization point to split-init this variable +variables.chpl:2: note: 'test' is used here before it is initialized +``` + +:::::::::::::::::::::::::::::::::::::::::::::::: + + + +In Chapel, to initialize a variable we must specify the type of the variable, or initialise it in place with some value. The common variable types in Chapel are: * integer `int` (positive or negative whole numbers) From 52437881fc8162c795f5efb45df7990f6ce98cfe Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 16:59:06 -0700 Subject: [PATCH 17/70] update the error message to 'cannot assign to const variable' --- episodes/02-variables.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index abff4f0..bd189f0 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -134,7 +134,7 @@ chpl variables.chpl -o variables.o ``` ```error -variables.chpl:2: error: illegal lvalue in assignment +variables.chpl:2: error: cannot assign to const variable ``` The compiler threw an error, and did not compile our program. This is a feature of compiled languages - if @@ -150,8 +150,8 @@ troubleshoot our programs. should carefully read the output and consider if it necessitates changing our code. Errors must be fixed, as they will block the code from compiling. -* `illegal lvalue in assignment` indicates that the left hand side of our assignment expression (`lvalue`) was - illegal. We were trying to reassign a `const` variable, which is explicitly not allowed in Chapel. +* `cannot assign to const variable` indicates that we were trying to reassign a `const` variable, which is + explicitly not allowed in Chapel. To fix this error, we can change `const` to `var` when declaring our `test` variable. `var` indicates a variable that can be reassigned. From c5f409d9a48f934c4d2483b34996d2970915c56f Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 17:12:37 -0700 Subject: [PATCH 18/70] floating-point division --- episodes/02-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index bd189f0..f2d4661 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -23,7 +23,7 @@ writeln(4 + 5); writeln(4 - 5); writeln(4 * 5); writeln(4 / 5); // integer division -writeln(4.0 / 5.0); // normal division +writeln(4.0 / 5.0); // floating-point division writeln(4 ** 5); // exponentiation ``` From 6a4301408beb758e12e5d6ac0892f736fb68a669 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 17:20:49 -0700 Subject: [PATCH 19/70] edit wording on memory --- episodes/02-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index f2d4661..85eb24c 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -59,7 +59,7 @@ It can span as many lines as you want! Granted, we probably want to do more than basic maths with Chapel. We will need to store the results of complex operations using variables. Variables in programming are not the same as the mathematical concept. In -programming, a variable is an allocated space in the memory of the computer, where we can store information or +programming, a variable represents (or references) a location in the memory of the computer where we can store information or data while executing a program. A variable has three elements: 1. a **_name_** or label, to identify the variable From c8f1e34009142cfcf80c9cd9d066817c60e789b0 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 18:18:49 -0700 Subject: [PATCH 20/70] change variable names: tt -> tmp, curdif -> delta, mindif -> tolerance, n -> outputFrequency --- episodes/02-variables.md | 18 ++++++------ episodes/03-ranges-arrays.md | 3 +- episodes/04-conditionals.md | 26 +++++++++-------- episodes/05-loops.md | 18 ++++++------ episodes/07-commandargs.md | 6 ++-- episodes/08-timing.md | 10 +++---- episodes/14-parallel-case-study.md | 46 +++++++++++++++--------------- 7 files changed, 65 insertions(+), 62 deletions(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index 85eb24c..3d00e98 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -182,15 +182,15 @@ default value depending on the declared type (0.0 for real variables, for exampl will be created as real floating point numbers equal to 0.0. ```chpl -var curdif: real; //here we will store the greatest difference in temperature from one iteration to another -var tt: real; //for temporary results when computing the temperatures +var delta: real; //here we will store the greatest difference in temperature from one iteration to another +var tmp: real; //for temporary results when computing the temperatures ``` Of course, we can use both, the initial value and the type, when declaring a variable as follows: ```chpl -const mindif=0.0001: real; //smallest difference in temperature that would be accepted before stopping -const n=20: int; //the temperature at the desired position will be printed every n iterations +const tolerance=0.0001: real; //smallest difference in temperature that would be accepted before stopping +const outputFrequency = 20: int; // the temperature will be printed every outputFrequency iterations ``` *This is not necessary, but it could help to make the code more readable.* @@ -202,13 +202,13 @@ examples, our simulation will be in the file `base_solution.chpl`. ```chpl const rows = 100; // number of rows in matrix const cols = 100; // number of columns in matrix -const niter = 500; // number of iterations +const niter = 500; // maximum number of iterations const x = 50; // row number of the desired position const y = 50; // column number of the desired position -var curdif: real; // here we will store the greatest difference in temperature from one iteration to another -var tt: real; // for temporary results when computing the temperatures -const mindif = 0.0001: real; // smallest difference in temperature that would be accepted before stopping -const n = 20: int; // the temperature at the desired position will be printed every n iterations +var delta: real; // here we will store the greatest difference in temperature from one iteration to another +var tmp: real; // for temporary results when computing the temperatures +const tolerance = 0.0001: real; // smallest difference in temperature that would be accepted before stopping +const outputFrequency = 20: int; // the temperature will be printed every outputFrequency iterations ``` ::::::::::::::::::::::::::::::::::::: keypoints diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index cf51e91..80c2548 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -168,7 +168,8 @@ const cols = 100; const niter = 500; const x = 50; // row number of the desired position const y = 50; // column number of the desired position -const mindif = 0.0001; // smallest difference in temperature that would be accepted before stopping +const tolerance = 0.0001; // smallest difference in temperature that would be accepted before stopping +const outputFrequency = 20: int; // the temperature will be printed every outputFrequency iterations // this is our "plate" var temp: [0..rows+1, 0..cols+1] real = 25; diff --git a/episodes/04-conditionals.md b/episodes/04-conditionals.md index d1d2507..acd987e 100644 --- a/episodes/04-conditionals.md +++ b/episodes/04-conditionals.md @@ -90,8 +90,8 @@ The main loop in our simulation can be programmed using a while statement like t ```chpl //this is the main loop of the simulation var c = 0; -var curdif = mindif; -while (c < niter && curdif >= mindif) do +var delta = tolerance; +while (c < niter && delta >= tolerance) do { c += 1; // actual simulation calculations will go here @@ -100,18 +100,19 @@ while (c < niter && curdif >= mindif) do Essentially, what we want is to repeat all the code inside the curly brackets until the number of iterations is greater than or equal to `niter`, or the difference of temperature between iterations is less than -`mindif`. (Note that in our case, as `curdif` was not initialised when declared -and thus Chapel assigned it +`tolerance`. (Note that in our case, as `delta` was not initialised when declared -and thus Chapel assigned it the default real value 0.0-, we need to assign it a value greater than or equal to 0.001, or otherwise the -condition of the while statement will never be satisfied. A good starting point is to simple say that `curdif` -is equal to `mindif`). +condition of the while statement will never be satisfied. A good starting point is to simple say that `delta` +is equal to `tolerance`). To count iterations we just need to keep adding 1 to the counter variable `c`. We could do this with `c=c+1`, or with the compound assignment, `+=`, as in the code above. To program the rest of the logic inside the curly brackets, on the other hand, we will need more elaborated instructions. -Let's focus, first, on printing the temperature every 20 iterations. To achieve this, we only need to check -whether `c` is a multiple of 20, and in that case, to print the temperature at the desired position. This is -the type of control that an **_if statement_** give us. The general syntax is: +Let's focus, first, on printing the temperature every `outputFrequency = 20` iterations. To achieve this, we +only need to check whether `c` is a multiple of `outputFrequency`, and in that case, to print the temperature +at the desired position. This is the type of control that an **_if statement_** give us. The general syntax +is: ```chpl if condition then @@ -126,7 +127,7 @@ executed otherwise (the else part of the if statement is optional). So, in our case this would do the trick: ```chpl -if (c % 20 == 0) +if (c % outputFrequency == 0) { writeln('Temperature at iteration ', c, ': ', temp[x, y]); } @@ -134,7 +135,7 @@ if (c % 20 == 0) Note that when only one instruction will be executed, there is no need to use the curly brackets. `%` is the modulo operator, it returns the remainder after the division (i.e. it returns zero when `c` is multiple of -20). +`outputFrequency`). Let's compile and execute our code to see what we get until now @@ -144,8 +145,9 @@ const cols = 100; const niter = 500; const x = 50; // row number of the desired position const y = 50; // column number of the desired position -const mindif = 0.0001; // smallest difference in temperature that +const tolerance = 0.0001; // smallest difference in temperature that // would be accepted before stopping +const outputFrequency = 20: int; // the temperature will be printed every outputFrequency iterations // this is our "plate" var temp: [0..rows+1, 0..cols+1] real = 25; @@ -158,7 +160,7 @@ var c = 0; while (c < niter) do { c += 1; - if (c % 20 == 0) + if (c % outputFrequency == 0) { writeln('Temperature at iteration ', c, ': ', temp[x, y]); } diff --git a/episodes/05-loops.md b/episodes/05-loops.md index 9119618..f4b9bac 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -220,31 +220,31 @@ Temperature at iteration 500: 0.823152 ## Challenge 3: Can you do it? -So far, `curdif` has been always equal to `mindif`, which means that our main while loop will always run the -500 iterations. So let's update `curdif` after each iteration. Use what we have studied so far to write the +So far, `delta` has been always equal to `tolerance`, which means that our main while loop will always run the +500 iterations. So let's update `delta` after each iteration. Use what we have studied so far to write the required piece of code. :::::::::::::::::::::::: solution The idea is simple, after each iteration of the while loop, we must compare all elements of `temp` and -`past_temp`, find the greatest difference, and update `curdif` with that value. The next nested for loops do +`past_temp`, find the greatest difference, and update `delta` with that value. The next nested for loops do the job: ```chpl -// update curdif, the greatest difference between temp and past_temp -curdif=0; +// update delta, the greatest difference between temp and past_temp +delta=0; for i in 1..rows do { for j in 1..cols do { - tt=abs(temp[i,j]-past_temp[i,j]); - if tt>curdif then curdif=tt; + tmp = abs(temp[i,j]-past_temp[i,j]); + if tmp > delta then delta = tmp; } } ``` Clearly there is no need to keep the difference at every single position in the array, we just need to update -`curdif` if we find a greater one. +`delta` if we find a greater one. ```bash chpl base_solution.chpl -o base_solution @@ -277,7 +277,7 @@ transfer equation. Let's just print some additional useful information, ```chpl // print final information writeln('\nFinal temperature at the desired position after ',c,' iterations is: ',temp[x,y]); -writeln('The difference in temperatures between the last two iterations was: ',curdif,'\n'); +writeln('The difference in temperatures between the last two iterations was: ',delta,'\n'); ``` and compile and execute our final code, diff --git a/episodes/07-commandargs.md b/episodes/07-commandargs.md index a663a26..0126e1b 100644 --- a/episodes/07-commandargs.md +++ b/episodes/07-commandargs.md @@ -13,7 +13,7 @@ exercises: 30 :::::::::::::::::::::::::::::::::::::::::::::::: From the last run of our code, we can see that 500 iterations is not enough to get to a _steady state_ (a -state where the difference in temperature does not vary too much, i.e. `curdif`<`mindif`). Now, if we want to +state where the difference in temperature does not vary too much, i.e. `delta`<`tolerance`). Now, if we want to change the number of iterations we would need to modify `niter` in the code, and compile it again. What if we want to change the number of rows and columns in our grid to have more precision, or if we want to see the evolution of the temperature at a different point (x,y)? The answer would be the same, modify the code and @@ -61,7 +61,7 @@ The greatest difference in temperatures between the last two iterations was: 0.0 ## Challenge 4: Can you do it? -Make `n`, `x`, `y`, `mindif`, `rows` and `cols` configurable variables, and test the code simulating different +Make `n`, `x`, `y`, `tolerance`, `rows` and `cols` configurable variables, and test the code simulating different configurations. What can you conclude about the performance of the code? :::::::::::::::::::::::: solution @@ -71,7 +71,7 @@ for 10000 iterations or until the difference of temperature between iterations i print the temperature every 1000 iterations. ```bash -./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 ``` ```output diff --git a/episodes/08-timing.md b/episodes/08-timing.md index c5e51ce..cffde39 100644 --- a/episodes/08-timing.md +++ b/episodes/08-timing.md @@ -20,7 +20,7 @@ But first, we need a quantitative way to measure the performance of our code. T see how long it takes to finish a simulation. The UNIX command `time` could be used to this effect ```bash -time ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +time ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 ``` ```output @@ -66,8 +66,8 @@ var watch: Timer; watch.start(); //this is the main loop of the simulation -curdif=mindif; -while (c=mindif) do +delta=tolerance; +while (c=tolerance) do { ... } @@ -77,12 +77,12 @@ watch.stop(); //print final information writeln('\nThe simulation took ',watch.elapsed(),' seconds'); writeln('Final temperature at the desired position after ',c,' iterations is: ',temp[x,y]); -writeln('The greatest difference in temperatures between the last two iterations was: ',curdif,'\n'); +writeln('The greatest difference in temperatures between the last two iterations was: ',delta,'\n'); ``` ```bash chpl base_solution.chpl -o base_solution -./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 ``` ```output diff --git a/episodes/14-parallel-case-study.md b/episodes/14-parallel-case-study.md index a5e4f6c..81cffd3 100644 --- a/episodes/14-parallel-case-study.md +++ b/episodes/14-parallel-case-study.md @@ -28,8 +28,8 @@ config const rowtasks = 2; config const coltasks = 2; // this is the main loop of the simulation -curdif = mindif; -while (c=mindif) do { +delta = tolerance; +while (c=tolerance) do { c += 1; coforall taskid in 0..coltasks*rowtasks-1 do { @@ -40,10 +40,10 @@ while (c=mindif) do { } } - curdif = max reduce (temp-past_temp); + delta = max reduce (temp-past_temp); past_temp = temp; - if c%n == 0 then writeln('Temperature at iteration ',c,': ',temp[x,y]); + if c%outputFrequency == 0 then writeln('Temperature at iteration ',c,': ',temp[x,y]); } ``` @@ -63,8 +63,8 @@ const nc = cols/coltasks; const rc = cols-nc*coltasks; // this is the main loop of the simulation -curdif = mindif; -while (c=mindif) do { +delta = tolerance; +while (c=tolerance) do { c+=1; coforall taskid in 0..coltasks*rowtasks-1 do { @@ -105,7 +105,7 @@ benchmark solution with our `coforall` parallelization to see how the performanc ```bash chpl --fast parallel_solution_1.chpl -o parallel1 -./parallel1 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +./parallel1 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 ``` ```output @@ -134,7 +134,7 @@ seconds needed by the benchmark solution, seems not very impressive. To understa the code's flow. When the program starts, the main thread does all the declarations and initialisations, and then, it enters the main loop of the simulation (the **_while loop_**). Inside this loop, the parallel tasks are launched for the first time. When these tasks finish their computations, the main task resumes its -execution, it updates `curdif`, and everything is repeated again. So, in essence, parallel tasks are launched +execution, it updates `delta`, and everything is repeated again. So, in essence, parallel tasks are launched and resumed 7750 times, which introduces a significant amount of overhead (the time the system needs to effectively start and destroy threads in the specific hardware, at each iteration of the while loop). @@ -151,7 +151,7 @@ const nc = cols/coltasks; const rc = cols-nc*coltasks; // this is the main loop of the simulation -curdif = mindif; +delta = tolerance; coforall taskid in 0..coltasks*rowtasks-1 do { var rowi, coli, rowf, colf: int; var taskr, taskc: int; @@ -178,7 +178,7 @@ coforall taskid in 0..coltasks*rowtasks-1 do { colf = (taskc*nc)+nc+rc; } - while (c=mindif) do { + while (c=tolerance) do { c = c+1; for i in rowi..rowf do { @@ -187,14 +187,14 @@ coforall taskid in 0..coltasks*rowtasks-1 do { } } - //update curdif + //update delta //update past_temp //print temperature in desired position } } ``` -The problem with this approach is that now we have to explicitly synchronise the tasks. Before, `curdif` and +The problem with this approach is that now we have to explicitly synchronise the tasks. Before, `delta` and `past_temp` were updated only by the main task at each iteration; similarly, only the main task was printing results. Now, all these operations must be carried inside the coforall loop, which imposes the need of synchronisation between tasks. @@ -202,25 +202,25 @@ synchronisation between tasks. The synchronisation must happen at two points: 1. We need to be sure that all tasks have finished with the computations of their part of the grid `temp`, - before updating `curdif` and `past_temp` safely. -2. We need to be sure that all tasks use the updated value of `curdif` to evaluate the condition of the while + before updating `delta` and `past_temp` safely. +2. We need to be sure that all tasks use the updated value of `delta` to evaluate the condition of the while loop for the next iteration. -To update `curdif` we could have each task computing the greatest difference in temperature in its associated +To update `delta` we could have each task computing the greatest difference in temperature in its associated sub-grid, and then, after the synchronisation, have only one task reducing all the sub-grids' maximums. ```chpl -var curdif: atomic real; +var delta: atomic real; var myd: [0..coltasks*rowtasks-1] real; ... //this is the main loop of the simulation -curdif.write(mindif); +delta.write(tolerance); coforall taskid in 0..coltasks*rowtasks-1 do { var myd2: real; ... - while (c=mindif) do { + while (c=tolerance) do { c = c+1; ... @@ -236,8 +236,8 @@ coforall taskid in 0..coltasks*rowtasks-1 do past_temp[rowi..rowf,coli..colf] = temp[rowi..rowf,coli..colf]; if taskid==0 then { - curdif.write(max reduce myd); - if c%n==0 then writeln('Temperature at iteration ',c,': ',temp[x,y]); + delta.write(max reduce myd); + if c%outputFrequency==0 then writeln('Temperature at iteration ',c,': ',temp[x,y]); } // here comes the synchronisation of tasks again @@ -261,11 +261,11 @@ var lock: atomic int; lock.write(0); ... //this is the main loop of the simulation -curdif.write(mindif); +delta.write(tolerance); coforall taskid in 0..coltasks*rowtasks-1 do { ... - while (c=mindif) do + while (c=tolerance) do { ... myd[taskid]=myd2 @@ -291,7 +291,7 @@ Using the solution in the Exercise 4, we can now compare the performance with th ```bash chpl --fast parallel_solution_2.chpl -o parallel2 -./parallel2 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --mindif=0.002 --n=1000 +./parallel2 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 ``` ```output From 0c328d3d665ff4052e60cd7903280e63079ebcb6 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 18:28:38 -0700 Subject: [PATCH 21/70] at compile-time --- episodes/02-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index 3d00e98..fef9b42 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -138,7 +138,7 @@ variables.chpl:2: error: cannot assign to const variable ``` The compiler threw an error, and did not compile our program. This is a feature of compiled languages - if -there is something wrong, we will typically see an error while writing our program, instead of while running +there is something wrong, we will typically see an error at compile-time, instead of while running it. Although we already kind of know why the error was caused (we tried to reassign the value of a `const` variable, which by definition cannot be changed), let's walk through the error as an example of how to troubleshoot our programs. From 899c5d61e97cb235c76f1ce5ef858d20ff414556 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Thu, 19 Sep 2024 18:44:58 -0700 Subject: [PATCH 22/70] mixed initialization and assignment: new wording, corrected syntax, added a call-out --- episodes/02-variables.md | 27 ++++++++++++++++++++++----- episodes/03-ranges-arrays.md | 2 +- episodes/04-conditionals.md | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index fef9b42..801decc 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -186,13 +186,30 @@ var delta: real; //here we will store the greatest difference in temperature fro var tmp: real; //for temporary results when computing the temperatures ``` -Of course, we can use both, the initial value and the type, when declaring a variable as follows: +When declaring a variable, we can assign its type in addition to its initial value: ```chpl -const tolerance=0.0001: real; //smallest difference in temperature that would be accepted before stopping -const outputFrequency = 20: int; // the temperature will be printed every outputFrequency iterations +const tolerance: real = 0.0001; //smallest difference in temperature that would be accepted before stopping +const outputFrequency: int = 20; // the temperature will be printed every outputFrequency iterations ``` + + +::::::::::::::::::::::::::::::::::::: callout + +Note that these two notations are different, but produce the same result in the end: + +```chpl +var a: real = 10.0; // we specify both the type and the value +var a = 10: real; // we specify only the value (10 converted to real) +``` + +:::::::::::::::::::::::::::::::::::::::::::::::: + + + + + *This is not necessary, but it could help to make the code more readable.* @@ -207,8 +224,8 @@ const x = 50; // row number of the desired position const y = 50; // column number of the desired position var delta: real; // here we will store the greatest difference in temperature from one iteration to another var tmp: real; // for temporary results when computing the temperatures -const tolerance = 0.0001: real; // smallest difference in temperature that would be accepted before stopping -const outputFrequency = 20: int; // the temperature will be printed every outputFrequency iterations +const tolerance: real = 0.0001; // smallest difference in temperature that would be accepted before stopping +const outputFrequency: int = 20; // the temperature will be printed every outputFrequency iterations ``` ::::::::::::::::::::::::::::::::::::: keypoints diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index 80c2548..5d9601d 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -169,7 +169,7 @@ const niter = 500; const x = 50; // row number of the desired position const y = 50; // column number of the desired position const tolerance = 0.0001; // smallest difference in temperature that would be accepted before stopping -const outputFrequency = 20: int; // the temperature will be printed every outputFrequency iterations +const outputFrequency: int = 20; // the temperature will be printed every outputFrequency iterations // this is our "plate" var temp: [0..rows+1, 0..cols+1] real = 25; diff --git a/episodes/04-conditionals.md b/episodes/04-conditionals.md index acd987e..6e9ab7e 100644 --- a/episodes/04-conditionals.md +++ b/episodes/04-conditionals.md @@ -147,7 +147,7 @@ const x = 50; // row number of the desired position const y = 50; // column number of the desired position const tolerance = 0.0001; // smallest difference in temperature that // would be accepted before stopping -const outputFrequency = 20: int; // the temperature will be printed every outputFrequency iterations +const outputFrequency: int = 20; // the temperature will be printed every outputFrequency iterations // this is our "plate" var temp: [0..rows+1, 0..cols+1] real = 25; From a6a92cfba44138e12da452fe2b2dab6a30d1ec2f Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 20 Sep 2024 19:13:23 -0700 Subject: [PATCH 23/70] rewrite 02-variables.md: first const vs. var, then initializing variables with type and/or value --- episodes/02-variables.md | 136 ++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 75 deletions(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index 801decc..37d046f 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -66,61 +66,9 @@ data while executing a program. A variable has three elements: 2. a **_type_**, that indicates the kind of data that we can store in it, and 3. a **_value_**, the actual information or data stored in the variable. -Variables in Chapel are declared with the `var` or `const` keywords. When a variable declared as const is -initialised, its value cannot be modified anymore during the execution of the program. - - - -::::::::::::::::::::::::::::::::::::: callout - -In the following code (saved as `variables.chpl`) we have not initialised the variable `test` before trying to -use it in line 2: - -```chpl -const test; // declare 'test' variable -writeln('The value of test is: ', test); -``` -```error -variables.chpl:1: error: 'test' is not initialized and has no type -variables.chpl:1: note: cannot find initialization point to split-init this variable -variables.chpl:2: note: 'test' is used here before it is initialized -``` - -:::::::::::::::::::::::::::::::::::::::::::::::: - - - -In Chapel, to initialize a variable we must specify the type of the variable, or initialise it in place with some -value. The common variable types in Chapel are: - -* integer `int` (positive or negative whole numbers) -* floating-point number `real` (decimal values) -* Boolean `bool` (true or false) -* string `string` (any type of text) - -If a variable is declared without a type, Chapel will infer it from the given -initial value. We can use the stored variable simply by using its name anywhere -in our code (called `variables.chpl`). - -```chpl -const test = 100; -writeln('The value of test is: ', test); -writeln(test / 4); -``` - -```bash -chpl variables.chpl -o variables.o --fast -./variables.o -``` - -```output -The value of test is: 100 -25 -``` - -This constant variable `test` will be created as an integer, and initialised with the value 100. No other -values can be assigned to these variables during the execution of the program. What happens if we try to -modify a constant variable like `test`? +Variables in Chapel are declared with the `var` or `const` keywords. When a variable declared as `const` is +initialised, its value cannot be modified anymore during the execution of the program. What happens if we try to +modify a constant variable like `test` below? ```chpl const test = 100; @@ -128,11 +76,9 @@ test = 200; writeln('The value of test is: ', test); writeln(test / 4); ``` - ```bash -chpl variables.chpl -o variables.o +chpl variables.chpl ``` - ```error variables.chpl:2: error: cannot assign to const variable ``` @@ -162,42 +108,68 @@ test = 200; writeln('The value of test is: ', test); writeln(test / 4); ``` - ```bash chpl variables.chpl -o variables.o ``` - ```output The value of test is: 200 50 ``` -It worked! Now we know both how to set, use, and change a variable, as well as the implications of using `var` -and `const`. We also know how to read and interpret errors. -## Uninitialised variables -On the other hand, if a variable is declared without an initial value, Chapel will initialise it with a -default value depending on the declared type (0.0 for real variables, for example). The following variables -will be created as real floating point numbers equal to 0.0. + + +In Chapel, to initialize a variable we must specify the type of the variable, or initialise it in place with +some value. The common variable types in Chapel are: + +* integer `int` (positive or negative whole numbers) +* floating-point number `real` (decimal values) +* Boolean `bool` (true or false) +* string `string` (any type of text) + +These two variables below are initialized with the type. If no initial value is given, Chapel will initialise +a variable with a default value depending on the declared type, for example 0 for integers and 0.0 for real +variables. ```chpl -var delta: real; //here we will store the greatest difference in temperature from one iteration to another -var tmp: real; //for temporary results when computing the temperatures +var counter: int; +var delta: real; +writeln("counter is ", counter, " and delta is ", delta); +``` +```bash +chpl variables.chpl +./variables.o +``` +```output +counter is 0 and delta is 0.0 ``` -When declaring a variable, we can assign its type in addition to its initial value: +If a variable is initialised with a value but without a type, Chapel will infer its type from the given +initial value: ```chpl -const tolerance: real = 0.0001; //smallest difference in temperature that would be accepted before stopping -const outputFrequency: int = 20; // the temperature will be printed every outputFrequency iterations +const test = 100; +writeln('The value of test is ', test, ' and its type is ', test.type:string); +``` +```bash +chpl variables.chpl +./variables.o +``` +```output +The value of test is 100 and its type is int(64) ``` +When initialising a variable, we can also assign its type in addition to its value: +```chpl +const tolerance: real = 0.0001; +const outputFrequency: int = 20; +``` ::::::::::::::::::::::::::::::::::::: callout -Note that these two notations are different, but produce the same result in the end: +Note that these two notations below are different, but produce the same result in the end: ```chpl var a: real = 10.0; // we specify both the type and the value @@ -207,14 +179,28 @@ var a = 10: real; // we specify only the value (10 converted to real) :::::::::::::::::::::::::::::::::::::::::::::::: +::::::::::::::::::::::::::::::::::::: callout +In the following code (saved as `variables.chpl`) we have not initialised the variable `test` before trying to +use it in line 2: +```chpl +const test; // declare 'test' variable +writeln('The value of test is: ', test); +``` +```error +variables.chpl:1: error: 'test' is not initialized and has no type +variables.chpl:1: note: cannot find initialization point to split-init this variable +variables.chpl:2: note: 'test' is used here before it is initialized +``` -*This is not necessary, but it could help to make the code more readable.* +:::::::::::::::::::::::::::::::::::::::::::::::: +Now we know how to set, use, and change a variable, as well as the implications of using `var` and `const`. We +also know how to read and interpret errors. -Let's practice defining variables and use this as the starting point of our simulation code. In these -examples, our simulation will be in the file `base_solution.chpl`. +Let's practice defining variables and use this as the starting point of our simulation code. The code will be +stored in the file `base_solution.chpl`. ```chpl const rows = 100; // number of rows in matrix From 5b6426c72e2fa690db4b1c7c9f60079a03b93764 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 20 Sep 2024 19:33:06 -0700 Subject: [PATCH 24/70] provide an explanation of the problem before introducing temperature-themed variable names --- episodes/02-variables.md | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index 37d046f..16dcc25 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -200,16 +200,29 @@ Now we know how to set, use, and change a variable, as well as the implications also know how to read and interpret errors. Let's practice defining variables and use this as the starting point of our simulation code. The code will be -stored in the file `base_solution.chpl`. +stored in the file `base_solution.chpl`. We will be solving the heat transfer problem introduced in the +previous section, starting with some initial temperature and computing a new temperature at each iteration. We +will then compute the greatest difference between the old and the new temperature and will check if it is +smaller than a preset `tolerance`. If no, we will continue iterating. If yes, we will stop iterations and will +print the final temperature. We will also stop iterations if we reach the maximum number of iterations +`niter`. + +Our grid will be of size `rows` by `cols`, and every `outputFrequency`th iteration we will print temperature +at coordinates `x` and `y`. + +The variable `delta` will store the greatest difference in temperature from one iteration to another. The +variable `tmp` will store some temporary results when computing the temperatures. + +Let's define our variables: ```chpl -const rows = 100; // number of rows in matrix -const cols = 100; // number of columns in matrix +const rows = 100; // number of rows in the grid +const cols = 100; // number of columns in the grid const niter = 500; // maximum number of iterations -const x = 50; // row number of the desired position -const y = 50; // column number of the desired position -var delta: real; // here we will store the greatest difference in temperature from one iteration to another -var tmp: real; // for temporary results when computing the temperatures +const x = 50; // row number for a printout +const y = 50; // column number for a printout +var delta: real; // greatest difference in temperature from one iteration to another +var tmp: real; // for temporary results const tolerance: real = 0.0001; // smallest difference in temperature that would be accepted before stopping const outputFrequency: int = 20; // the temperature will be printed every outputFrequency iterations ``` From 3dd4deb3aebf418eb2befe9f6e0d475b7cff475c Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 20 Sep 2024 19:49:57 -0700 Subject: [PATCH 25/70] change the keypoint on const vs. var variables --- episodes/02-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index 16dcc25..76e72d5 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -230,5 +230,5 @@ const outputFrequency: int = 20; // the temperature will be printed every outp ::::::::::::::::::::::::::::::::::::: keypoints - "A comment is preceded with `//` or surrounded by `/* and `*/`" - "All variables hold a certain type of data." -- "Using `const` instead of `var` prevents reassignment." +- "Reassigning a new value to a `const` variable will produce an error during compilation. If you want to assign a new value to a variable, declare that variable with the `var` keyword." :::::::::::::::::::::::::::::::::::::::::::::::: From 048fe12bf790f5de6f996e00c1b64a6a0ee04bfa Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Sat, 21 Sep 2024 14:52:58 -0700 Subject: [PATCH 26/70] correct a typo --- episodes/05-loops.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/05-loops.md b/episodes/05-loops.md index f4b9bac..907bada 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -5,7 +5,7 @@ exercises: 30 --- :::::::::::::::::::::::::::::::::::::: questions -- "How do I get run the same piece of code repeatedly?" +- "How do I run the same piece of code repeatedly?" :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives From dac45dc50ed61590c5f42e36e50c8d82fb59b4ad Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Sat, 21 Sep 2024 15:06:41 -0700 Subject: [PATCH 27/70] added three key points --- episodes/04-conditionals.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/episodes/04-conditionals.md b/episodes/04-conditionals.md index 6e9ab7e..d594cbd 100644 --- a/episodes/04-conditionals.md +++ b/episodes/04-conditionals.md @@ -206,5 +206,11 @@ Of course the temperature is always 25.0 at any iteration other than the initial computation yet. ::::::::::::::::::::::::::::::::::::: keypoints -- "Conditional statements in Chapel are very similar to these in other languages." +- "Use `if then {instructions A} else {instructions B}` syntax to execute one set of instructions + if the condition is satisfied, and the other set of instructions if the condition is not satisfied." +- This syntax can be simplified to `if then {instructions}` if we only want to execute the + instructions within the curly brackets if the condition is satisfied. +- "Use `while do {instructions}` to repeatedly execute the instructions within the curly brackets + while the condition is satisfied. The instructions will be executed over and over again until the condition + does not hold anymore." :::::::::::::::::::::::::::::::::::::::::::::::: From 26390708cc1b97ba8df1b884f028177e4e8e8203 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Sat, 21 Sep 2024 15:27:57 -0700 Subject: [PATCH 28/70] correct wording around arrays (values need not be sequential) --- episodes/03-ranges-arrays.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index 5d9601d..ac3a4c1 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -33,7 +33,8 @@ chpl ranges.chpl -o ranges.o Our example range was set to: 0..10 ``` -An array is a multidimensional sequence of values. Arrays can be any size, and are defined using ranges: Let's +An array is a multidimensional collection of values of the same type. Arrays can be any size, and are defined +using ranges. Let's define a 1-dimensional array of the size `example_range` and see what it looks like. Notice how the size of an array is included with its type. @@ -45,7 +46,7 @@ writeln('Our example array is now: ', example_array); ``` We can reassign the values in our example array the same way we would reassign a variable. An array can either -be set all to a single value, or a sequence of values. +be set all to a single value, or to a sequence of values. ```chpl var example_range = 0..10; @@ -190,7 +191,7 @@ Temperature at start is: 25.0 ::::::::::::::::::::::::::::::::::::: keypoints - "A range is a sequence of integers." -- "An array holds a sequence of values." +- "An array holds a non-negative number of values of the same type." - "Chapel arrays can start at any index, not just 0 or 1." - "You can index arrays with the `[]` brackets." :::::::::::::::::::::::::::::::::::::::::::::::: From 894963e35e2b5734759fd5af13b02fbd4e5e3d02 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Sun, 22 Sep 2024 21:30:49 -0700 Subject: [PATCH 29/70] describe the ghost layer; set the initial temperature there to 0.0 --- episodes/03-ranges-arrays.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index ac3a4c1..d3d9be2 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -146,19 +146,25 @@ When set to a range: 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 10.0 11.0 ## Back to our simulation -Let's define a two dimensional array for use in our simulation: +Let's define a two-dimensional array for use in our simulation and set its initial values: ```chpl // this is our "plate" -var temp: [0..rows+1, 0..cols+1] real = 25; +var temp: [0..rows+1, 0..cols+1] real; +temp[1..rows,1..cols] = 25; // set the initial temperature on the internal grid ``` -This is a matrix (2D array) with (`rows + 2`) rows and (`cols + 2`) columns of real numbers, all initialised -as 25.0. The ranges `0..rows+1` and `0..cols+1` used here, not only define the size and shape of the array, -they stand for the indices with which we could access particular elements of the array using the `[ , ]` -notation. For example, `temp[0,0]` is the real variable located at the first row and first column of the array -`temp`, while `temp[3,7]` is the one at the 4th row and 8th column; `temp[2,3..15]` access columns 4th to 16th -of the 3th row of `temp`, and `temp[0..3,4]` corresponds to the first 4 rows on the 5th column of `temp`. +This is a matrix (2D array) with (`rows + 2`) rows and (`cols + 2`) columns of real numbers. The ranges +`0..rows+1` and `0..cols+1` used here, not only define the size and shape of the array, they stand for the +indices with which we could access particular elements of the array using the `[ , ]` notation. For example, +`temp[0,0]` is the real variable located at the first row and first column of the array `temp`, while +`temp[3,7]` is the one at the 4th row and 8th column; `temp[2,3..15]` access columns 4th to 16th of the 3th +row of `temp`, and `temp[0..3,4]` corresponds to the first 4 rows on the 5th column of `temp`. + +We divide our "plate" into two parts: (1) the internal grid `1..rows,1..cols` on which we set the initial +temperature at 25.0, and (2) the surrounding layer of *ghost points* with row indices equal to `0` or `rows+1` +and column indices equal to `0` or `cols+1`. The temperature in the ghost layer is equal to 0.0 by default, as +we do not assign a value there. We must now be ready to start coding our simulations. Let's print some information about the initial configuration, compile the code, and execute it to see if everything is working as expected. @@ -173,7 +179,8 @@ const tolerance = 0.0001; // smallest difference in temperature that would const outputFrequency: int = 20; // the temperature will be printed every outputFrequency iterations // this is our "plate" -var temp: [0..rows+1, 0..cols+1] real = 25; +var temp: [0..rows+1, 0..cols+1] real; +temp[1..rows,1..cols] = 25; // set the initial temperature on the internal grid writeln('This simulation will consider a matrix of ', rows, ' by ', cols, ' elements.'); writeln('Temperature at start is: ', temp[x, y]); From 8876d3d214115d9781299f7fe9e896bbff29e434 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Sun, 22 Sep 2024 21:38:43 -0700 Subject: [PATCH 30/70] rewrite the paragraph about points close to the boundary --- episodes/05-loops.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/episodes/05-loops.md b/episodes/05-loops.md index 907bada..8820805 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -14,7 +14,7 @@ exercises: 30 To compute the current temperature of an element of `temp`, we need to add all the surrounding elements in `past_temp`, and divide the result by 4. And, essentially, we need to repeat this process for all the elements -of `temp`, or, in other words, we need to *iterate* over the elements of `temp`. When it comes to iterate over +of `temp`, or, in other words, we need to *iterate* over the elements of `temp`. When it comes to iterating over a given number of elements, the **_for-loop_** is what we want to use. The for-loop has the following general syntax: @@ -126,9 +126,12 @@ is cooling down. ## Challenge 1: Can you do it? -What would be the temperature at the top right corner of the plate? The border of the plate is in contact with -the boundary conditions, which are set to zero, so we expect the temperature at these points to decrease -faster. Modify the code to see the temperature at the top right corner. +What would be the temperature at the top right corner of the plate? In our current setup we have a layer of +ghost points around the internal grid. While the temperature on the internal grid was initially set to 25.0, +the temperature at the ghost points was set to 0.0. Note that during our iterations we do not compute the +temperature at the ghost points -- it is permanently set to 0.0. Consequently, any point close to the ghost +layer will be influenced by this zero temperature, so we expect the temperature near the border of the plate +to decrease faster. Modify the code to see the temperature at the top right corner. :::::::::::::::::::::::: solution From b34c8cdedf72062e973ede4dc8c544f708b85dbc Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Sun, 22 Sep 2024 22:27:35 -0700 Subject: [PATCH 31/70] use consistent language with temp and temp_new throughout all chapters --- episodes/01-intro.md | 6 ++-- episodes/05-loops.md | 53 +++++++++++++----------------- episodes/14-parallel-case-study.md | 34 +++++++++---------- episodes/22-domains.md | 50 ++++++++++++++-------------- 4 files changed, 68 insertions(+), 75 deletions(-) diff --git a/episodes/01-intro.md b/episodes/01-intro.md index d401897..4f92546 100644 --- a/episodes/01-intro.md +++ b/episodes/01-intro.md @@ -110,10 +110,10 @@ points, and to evaluate the temperature on each point at each iteration, accordi **_difference equation_**: ```chpl -T[i,j] = 0.25 * (Tp[i-1,j] + Tp[i+1,j] + Tp[i,j-1] + Tp[i,j+1]) +temp_new[i,j] = 0.25 * (temp[i-1,j] + temp[i+1,j] + temp[i,j-1] + temp[i,j+1]) ``` -Here `T` stands for the temperature at the current iteration, while `Tp` contains the temperature calculated +Here `temp_new` stands for the new temperature at the current iteration, while `temp` contains the temperature calculated at the past iteration (or the initial conditions in case we are at the first iteration). The indices `i` and `j` indicate that we are working on the point of the grid located at the *i*th row and the *j*th column. @@ -125,7 +125,7 @@ So, our objective is to: > > * It should work for any given number of rows and columns in the grid. > * It should run for a given number of iterations, or until the difference -> between `T` and `Tp` is smaller than a given tolerance value. +> between `temp_new` and `temp` is smaller than a given tolerance value. > * It should output the temperature at a desired position on the grid every > given number of iterations. > diff --git a/episodes/05-loops.md b/episodes/05-loops.md index 8820805..27942c1 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -12,9 +12,9 @@ exercises: 30 - "First objective." :::::::::::::::::::::::::::::::::::::::::::::::: -To compute the current temperature of an element of `temp`, we need to add all the surrounding elements in -`past_temp`, and divide the result by 4. And, essentially, we need to repeat this process for all the elements -of `temp`, or, in other words, we need to *iterate* over the elements of `temp`. When it comes to iterating over +To compute the new temperature, i.e. each element of `temp_new`, we need to add all the surrounding elements in +`temp` and divide the result by 4. And, essentially, we need to repeat this process for all the elements +of `temp_new`, or, in other words, we need to *iterate* over the elements of `temp_new`. When it comes to iterating over a given number of elements, the **_for-loop_** is what we want to use. The for-loop has the following general syntax: @@ -33,18 +33,19 @@ again. This pattern is repeated until index takes all the different values expre This for loop, for example ```chpl -// calculate the new current temperatures (temp) using the past temperatures (past_temp) +// calculate the new temperatures (temp_new) using the past temperatures (temp) for i in 1..rows do { // do this for every row } ``` -will allow us to iterate over the rows of `temp`. Now, for each row we also need to iterate over all the -columns in order to access every single element of `temp`. This can be done with nested for loops like this +will allow us to iterate over the rows of `temp_new`. Now, for each row we also need to iterate over all the +columns in order to access every single element of `temp_new`. This can be done with nested `for` loops like +this: ```chpl -// calculate the new current temperatures (temp) using the past temperatures (past_temp) +// calculate the new temperatures (temp_new) using the past temperatures (temp) for i in 1..rows do { // do this for every row @@ -59,21 +60,21 @@ Now, inside the inner loop, we can use the indices `i` and `j` to perform the re follows: ```chpl -// calculate the new current temperatures (temp) using the past temperatures (past_temp) +// calculate the new temperatures (temp_new) using the past temperatures (temp) for i in 1..rows do { // do this for every row for j in 1..cols do { // and this for every column in the row i - temp[i,j]=(past_temp[i-1,j]+past_temp[i+1,j]+past_temp[i,j-1]+past_temp[i,j+1])/4; + temp_new[i,j] = (temp[i-1,j] + temp[i+1,j] + temp[i,j-1] + temp[i,j+1]) / 4; } } -past_temp=temp; +temp=temp_new; ``` -Note that at the end of the outer for-loop, when all the elements in `temp` are already calculated, we update -`past_temp` with the values of `temp`; this way everything is set up for the next iteration of the main while +Note that at the end of the outer `for` loop, when all the elements in `temp_new` are already calculated, we update +`temp` with the values of `temp_new`; this way everything is set up for the next iteration of the main `while` statement. Now let's compile and execute our code again: @@ -174,26 +175,18 @@ boundary conditions. Compile and run your code to see how the temperature is cha :::::::::::::::::::::::: solution To get the linear distribution, the 80 degrees must be divided by the number of rows or columns in our -plate. So, the following couple of for loops will give us what we want: +plate. So, the following couple of `for` loops at the start of time iteration will give us what we want: ```chpl -// this setup the boundary conditions +// set the boundary conditions for i in 1..rows do -{ - past_temp[i,cols+1]=i*80.0/rows; - temp[i,cols+1]=i*80.0/rows; -} + temp[i,cols+1] = i*80.0/rows; // right side for j in 1..cols do -{ - past_temp[rows+1,j]=j*80.0/cols; - temp[rows+1,j]=j*80.0/cols; -} + temp[rows+1,j] = j*80.0/cols; // bottom side ``` -Note that the boundary conditions must be set in both arrays, `past_temp` and `temp`, otherwise, they will be -set to zero again after the first iteration. Also note that 80 degrees are written as a real numbe r 80.0. -The division of integers in Chapel returns an integer, then, as `rows` and `cols` are integers, we must have -80 as real so that the quotient is not truncated. +Note that 80 degrees is written as a real number 80.0. The division of integers in Chapel returns an integer, +then, as `rows` and `cols` are integers, we must have 80 as real so that the result is not truncated. ```bash chpl base_solution.chpl -o base_solution @@ -229,18 +222,18 @@ required piece of code. :::::::::::::::::::::::: solution -The idea is simple, after each iteration of the while loop, we must compare all elements of `temp` and -`past_temp`, find the greatest difference, and update `delta` with that value. The next nested for loops do +The idea is simple, after each iteration of the while loop, we must compare all elements of `temp_new` and +`temp`, find the greatest difference, and update `delta` with that value. The next nested for loops do the job: ```chpl -// update delta, the greatest difference between temp and past_temp +// update delta, the greatest difference between temp_new and temp delta=0; for i in 1..rows do { for j in 1..cols do { - tmp = abs(temp[i,j]-past_temp[i,j]); + tmp = abs(temp_new[i,j]-temp[i,j]); if tmp > delta then delta = tmp; } } diff --git a/episodes/14-parallel-case-study.md b/episodes/14-parallel-case-study.md index 81cffd3..6c2c170 100644 --- a/episodes/14-parallel-case-study.md +++ b/episodes/14-parallel-case-study.md @@ -16,8 +16,8 @@ Here is our plan to task-parallelize the heat transfer equation: 1. divide the entire grid of points into blocks and assign blocks to individual tasks, 1. each task should compute the new temperature of its assigned points, -1. perform a **_reduction_** over the whole grid, to update the greatest temperature difference between `Tnew` - and `T`. +1. perform a **_reduction_** over the whole grid, to update the greatest temperature difference between + `temp_new` and `temp`. For the reduction of the grid we can simply use the `max reduce` statement, which is already parallelized. Now, let's divide the grid into `rowtasks` x `coltasks` sub-grids, and assign each sub-grid to a @@ -35,13 +35,13 @@ while (c=tolerance) do { coforall taskid in 0..coltasks*rowtasks-1 do { for i in rowi..rowf do { for j in coli..colf do { - temp[i,j] = (past_temp[i-1,j]+past_temp[i+1,j]+past_temp[i,j-1]+past_temp[i,j+1]) / 4; + temp_new[i,j] = (temp[i-1,j] + temp[i+1,j] + temp[i,j-1] + temp[i,j+1]) / 4; } } } - delta = max reduce (temp-past_temp); - past_temp = temp; + delta = max reduce (temp_new-temp); + temp = temp_new; if c%outputFrequency == 0 then writeln('Temperature at iteration ',c,': ',temp[x,y]); } @@ -183,26 +183,26 @@ coforall taskid in 0..coltasks*rowtasks-1 do { for i in rowi..rowf do { for j in coli..colf do { - temp[i,j] = (past_temp[i-1,j]+past_temp[i+1,j]+past_temp[i,j-1]+past_temp[i,j+1])/4; + temp_new[i,j] = (temp[i-1,j] + temp[i+1,j] + temp[i,j-1] + temp[i,j+1]) / 4; } } //update delta - //update past_temp + //update temp //print temperature in desired position } } ``` The problem with this approach is that now we have to explicitly synchronise the tasks. Before, `delta` and -`past_temp` were updated only by the main task at each iteration; similarly, only the main task was printing +`temp` were updated only by the main task at each iteration; similarly, only the main task was printing results. Now, all these operations must be carried inside the coforall loop, which imposes the need of synchronisation between tasks. The synchronisation must happen at two points: 1. We need to be sure that all tasks have finished with the computations of their part of the grid `temp`, - before updating `delta` and `past_temp` safely. + before updating `delta` and `temp` safely. 2. We need to be sure that all tasks use the updated value of `delta` to evaluate the condition of the while loop for the next iteration. @@ -226,15 +226,15 @@ coforall taskid in 0..coltasks*rowtasks-1 do for i in rowi..rowf do { for j in coli..colf do { - temp[i,j] = (past_temp[i-1,j]+past_temp[i+1,j]+past_temp[i,j-1]+past_temp[i,j+1])/4; - myd2 = max(abs(temp[i,j]-past_temp[i,j]),myd2); + temp_new[i,j] = (temp[i-1,j] + temp[i+1,j] + temp[i,j-1] + temp[i,j+1]) / 4; + myd2 = max(abs(temp_new[i,j]-temp[i,j]),myd2); } } myd[taskid] = myd2 // here comes the synchronisation of tasks - past_temp[rowi..rowf,coli..colf] = temp[rowi..rowf,coli..colf]; + temp[rowi..rowf,coli..colf] = temp_new[rowi..rowf,coli..colf]; if taskid==0 then { delta.write(max reduce myd); if c%outputFrequency==0 then writeln('Temperature at iteration ',c,': ',temp[x,y]); @@ -274,7 +274,7 @@ coforall taskid in 0..coltasks*rowtasks-1 do lock.add(1); lock.waitFor(coltasks*rowtasks); - past_temp[rowi..rowf,coli..colf]=temp[rowi..rowf,coli..colf]; + temp[rowi..rowf,coli..colf] = temp_new[rowi..rowf,coli..colf]; ... //here comes the synchronisation of tasks again @@ -322,18 +322,18 @@ stepping) using data parallelism on a single locale: ```chpl const n = 100, stride = 20; -var T: [0..n+1, 0..n+1] real; -var Tnew: [1..n,1..n] real; +var temp: [0..n+1, 0..n+1] real; +var temp_new: [1..n,1..n] real; var x, y: real; for (i,j) in {1..n,1..n} { // serial iteration x = ((i:real)-0.5)/n; y = ((j:real)-0.5)/n; - T[i,j] = exp(-((x-0.5)**2 + (y-0.5)**2)/0.01); // narrow Gaussian peak + temp[i,j] = exp(-((x-0.5)**2 + (y-0.5)**2)/0.01); // narrow Gaussian peak } coforall (i,j) in {1..n,1..n} by (stride,stride) { // 5x5 decomposition into 20x20 blocks => 25 tasks for k in i..i+stride-1 { // serial loop inside each block for l in j..j+stride-1 do { - Tnew[k,l] = (T[k-1,l] + T[k+1,l] + T[k,l-1] + T[k,l+1]) / 4; + temp_new[k,l] = (temp[k-1,l] + temp[k+1,l] + temp[k,l-1] + temp[k,l+1]) / 4; } } } diff --git a/episodes/22-domains.md b/episodes/22-domains.md index 95eaa94..e8c59c1 100644 --- a/episodes/22-domains.md +++ b/episodes/22-domains.md @@ -299,17 +299,17 @@ const mesh: domain(2) = {1..n, 1..n}; // local 2D n^2 domain ``` We will add a larger (n+2)^2 block-distributed domain `largerMesh` with a layer of *ghost points* on -*perimeter locales*, and define a temperature array T on top of it, by adding the following to our code: +*perimeter locales*, and define a temperature array `temp` on top of it, by adding the following to our code: ```chpl const largerMesh: domain(2) dmapped Block(boundingBox=mesh) = {0..n+1, 0..n+1}; -var T: [largerMesh] real; // a block-distributed array of temperatures +var temp: [largerMesh] real; // a block-distributed array of temperatures forall (i,j) in T.domain[1..n,1..n] { var x = ((i:real)-0.5)/(n:real); // x, y are local to each task var y = ((j:real)-0.5)/(n:real); - T[i,j] = exp(-((x-0.5)**2 + (y-0.5)**2) / 0.01); // narrow Gaussian peak + temp[i,j] = exp(-((x-0.5)**2 + (y-0.5)**2) / 0.01); // narrow Gaussian peak } -writeln(T); +writeln(temp); ``` Here we initialised an initial Gaussian temperature peak in the middle of the mesh. As we evolve our solution @@ -340,7 +340,7 @@ The code above will print the initial temperature distribution: 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ``` -Let us define an array of strings `nodeID` with the same distribution over locales as T, by adding the +Let us define an array of strings `nodeID` with the same distribution over locales as `temp`, by adding the following to our code: ```chpl @@ -381,11 +381,11 @@ Something along the lines: `m = here.id:string + '-' + m.locale.id:string` Now we implement the parallel solver, by adding the following to our code (*contains a mistake on purpose!*): ```chpl -var Tnew: [largerMesh] real; +var temp_new: [largerMesh] real; for step in 1..5 { // time-stepping forall (i,j) in mesh do - Tnew[i,j] = (T[i-1,j] + T[i+1,j] + T[i,j-1] + T[i,j+1]) / 4; - T[mesh] = Tnew[mesh]; // uses parallel forall underneath + temp_new[i,j] = (temp[i-1,j] + temp[i+1,j] + temp[i,j-1] + temp[i,j+1]) / 4; + temp[mesh] = temp_new[mesh]; // uses parallel forall underneath } ``` @@ -399,7 +399,7 @@ Can anyone spot a mistake in the last code? It should be -`forall (i,j) in Tnew.domain[1..n,1..n] do` +`forall (i,j) in temp_new.domain[1..n,1..n] do` instead of @@ -418,30 +418,30 @@ use BlockDist; config const n = 8; const mesh: domain(2) = {1..n,1..n}; const largerMesh: domain(2) dmapped Block(boundingBox=mesh) = {0..n+1,0..n+1}; -var T, Tnew: [largerMesh] real; -forall (i,j) in T.domain[1..n,1..n] { +var temp, temp_new: [largerMesh] real; +forall (i,j) in temp.domain[1..n,1..n] { var x = ((i:real)-0.5)/(n:real); var y = ((j:real)-0.5)/(n:real); - T[i,j] = exp(-((x-0.5)**2 + (y-0.5)**2) / 0.01); + temp[i,j] = exp(-((x-0.5)**2 + (y-0.5)**2) / 0.01); } for step in 1..5 { - forall (i,j) in Tnew.domain[1..n,1..n] { - Tnew[i,j] = (T[i-1,j]+T[i+1,j]+T[i,j-1]+T[i,j+1])/4.0; + forall (i,j) in temp_new.domain[1..n,1..n] { + temp_new[i,j] = (temp[i-1,j] + temp[i+1,j] + temp[i,j-1] + temp[i,j+1]) / 4.0; } - T = Tnew; - writeln((step,T[n/2,n/2],T[1,1])); + temp = temp_new; + writeln((step, " ", temp[n/2,n/2], " ", temp[1,1])); } ``` -This is the entire parallel solver! Note that we implemented an open boundary: T on *ghost points* is +This is the entire parallel solver! Note that we implemented an open boundary: `temp` on the *ghost points* is always 0. Let us add some printout and also compute the total energy on the mesh, by adding the following to our code: ```chpl - writeln((step, T[n/2,n/2], T[2,2])); + writeln((step, " ", temp[n/2,n/2], " ", temp[2,2])); var total: real = 0; forall (i,j) in mesh with (+ reduce total) do - total += T[i,j]; + total += temp[i,j]; writeln("total = ", total); ``` @@ -454,7 +454,7 @@ system. ## Challenge 5: Can you do it? Write a code to print how the finite-difference stencil [i,j], [i-1,j], [i+1,j], [i,j-1], [i,j+1] is -distributed among nodes, and compare that to the ID of the node where T[i,i] is computed. +distributed among nodes, and compare that to the ID of the node where temp[i,i] is computed. :::::::::::::::::::::::: solution @@ -487,7 +487,7 @@ empty 222222 222222 222222 222223 333323 333333 333333 333333 em empty empty empty empty empty empty empty empty empty empty ``` -Note that T[i,j] is always computed on the same node where that element is stored, which makes sense. +Note that temp[i,j] is always computed on the same node where that element is stored, which makes sense. ## Periodic boundary conditions @@ -496,10 +496,10 @@ need to set elements on the *ghost points* to their respective values on the *op following to our code: ```chpl - T[0,1..n] = T[n,1..n]; // periodic boundaries on all four sides; these will run via parallel forall - T[n+1,1..n] = T[1,1..n]; - T[1..n,0] = T[1..n,n]; - T[1..n,n+1] = T[1..n,1]; + temp[0,1..n] = temp[n,1..n]; // periodic boundaries on all four sides; these will run via parallel forall + temp[n+1,1..n] = temp[1,1..n]; + temp[1..n,0] = temp[1..n,n]; + temp[1..n,n+1] = temp[1..n,1]; ``` Now total energy should be conserved, as nothing leaves the domain. From e53093a1846b076470fd5e803bc70f0fceba6bcb Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 08:31:50 -0700 Subject: [PATCH 32/70] typo correction --- episodes/04-conditionals.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/episodes/04-conditionals.md b/episodes/04-conditionals.md index d594cbd..10ecf92 100644 --- a/episodes/04-conditionals.md +++ b/episodes/04-conditionals.md @@ -100,8 +100,8 @@ while (c < niter && delta >= tolerance) do Essentially, what we want is to repeat all the code inside the curly brackets until the number of iterations is greater than or equal to `niter`, or the difference of temperature between iterations is less than -`tolerance`. (Note that in our case, as `delta` was not initialised when declared -and thus Chapel assigned it -the default real value 0.0-, we need to assign it a value greater than or equal to 0.001, or otherwise the +`tolerance`. (Note that in our case, as `delta` was not initialised when declared -- and thus Chapel assigned it +the default real value 0.0 -- we need to assign it a value greater than or equal to 0.001, or otherwise the condition of the while statement will never be satisfied. A good starting point is to simple say that `delta` is equal to `tolerance`). From a8d1aeb11322c27d650bdd8381edcfd1a3017a23 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 09:39:59 -0700 Subject: [PATCH 33/70] increase niter to 10_000 so that we reach convergence after 7505 iterations; update this chapter accordingly --- episodes/04-conditionals.md | 5 +++-- episodes/05-loops.md | 24 ++++++++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/episodes/04-conditionals.md b/episodes/04-conditionals.md index 10ecf92..42c133e 100644 --- a/episodes/04-conditionals.md +++ b/episodes/04-conditionals.md @@ -90,7 +90,7 @@ The main loop in our simulation can be programmed using a while statement like t ```chpl //this is the main loop of the simulation var c = 0; -var delta = tolerance; +delta = tolerance; while (c < niter && delta >= tolerance) do { c += 1; @@ -157,7 +157,8 @@ writeln('Temperature at start is: ', temp[x, y]); //this is the main loop of the simulation var c = 0; -while (c < niter) do +delta = tolerance; +while (c < niter && delta >= tolerance) do { c += 1; if (c % outputFrequency == 0) diff --git a/episodes/05-loops.md b/episodes/05-loops.md index 27942c1..09d19d7 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -216,8 +216,17 @@ Temperature at iteration 500: 0.823152 ## Challenge 3: Can you do it? -So far, `delta` has been always equal to `tolerance`, which means that our main while loop will always run the -500 iterations. So let's update `delta` after each iteration. Use what we have studied so far to write the +Let us increase the maximum number of iterations to `niter = 10_000`. The code now does 10_000 iterations: + +```output +... +Temperature at iteration 9960: 0.79214 +Temperature at iteration 9980: 0.792139 +Temperature at iteration 10000: 0.792139 +``` + +So far, `delta` has been always equal to `tolerance`, which means that our main `while` loop will always run +`niter` iterations. So let's update `delta` after each iteration. Use what we have studied so far to write the required piece of code. :::::::::::::::::::::::: solution @@ -249,7 +258,7 @@ chpl base_solution.chpl -o base_solution ```output The simulation will consider a matrix of 100 by 100 elements, -it will run up to 500 iterations, or until the largest difference +it will run up to 10000 iterations, or until the largest difference in temperature between iterations is less than 0.0001. You are interested in the evolution of the temperature at the position (1,100) of the matrix... @@ -259,9 +268,12 @@ Temperature at iteration 0: 25.0 Temperature at iteration 20: 2.0859 Temperature at iteration 40: 1.42663 ... -Temperature at iteration 460: 0.826941 -Temperature at iteration 480: 0.824959 -Temperature at iteration 500: 0.823152 +Temperature at iteration 7460: 0.792283 +Temperature at iteration 7480: 0.792281 +Temperature at iteration 7500: 0.792279 + +Final temperature at the desired position after 7505 iterations is: 0.792279 +The difference in temperatures between the last two iterations was: 9.99834e-05 ``` ::::::::::::::::::::::::::::::::: From 5b84e13793fb207296837e00cc0fb3cb3352f44f Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 09:55:35 -0700 Subject: [PATCH 34/70] rewrite the key point discussing both for and while loops --- episodes/05-loops.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/episodes/05-loops.md b/episodes/05-loops.md index 09d19d7..0f0465c 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -30,7 +30,7 @@ iterand, and keeps it until all the instructions inside the curly brackets are e index takes the second value yielded by the iterand, and keeps it until all the instructions are executed again. This pattern is repeated until index takes all the different values expressed by the iterand. -This for loop, for example +This `for` loop, for example ```chpl // calculate the new temperatures (temp_new) using the past temperatures (temp) @@ -335,5 +335,9 @@ The greatest difference in temperatures between the last two iterations was: 0.0 ``` ::::::::::::::::::::::::::::::::::::: keypoints -- "Use `for` statement to organise a loop." +- "You can organize loops with `for` and `while` statements. Use a `for` loop to run over every element of the + iterand, e.g. `for i in 1..rows do { ...}` will run over all integers from 1 to `rows`. Use a `while` + statement to repeatedly execute a code block until the condition does not hold anymore, e.g. `while (c < + niter && delta >= tolerance) do {...}` will repeatedly execute the commands in curly braces until one of the + two conditions turns false." :::::::::::::::::::::::::::::::::::::::::::::::: From 2f082675767aff25b042f5318ebeb0aa03862d23 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 10:00:33 -0700 Subject: [PATCH 35/70] remove 'functional programming' --- episodes/06-procedures.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/06-procedures.md b/episodes/06-procedures.md index 762e13a..839e57f 100644 --- a/episodes/06-procedures.md +++ b/episodes/06-procedures.md @@ -1,5 +1,5 @@ --- -title: "Procedures for functional programming" +title: "Procedures" teaching: 15 exercises: 0 --- From fe26452604204cb708aa17d6f2e4d67b6c85e55d Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 10:43:13 -0700 Subject: [PATCH 36/70] add a description of the tuple example in 06-procedures.md; change *variable* to *varying* --- episodes/06-procedures.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/episodes/06-procedures.md b/episodes/06-procedures.md index 839e57f..171fc95 100644 --- a/episodes/06-procedures.md +++ b/episodes/06-procedures.md @@ -32,7 +32,10 @@ proc fibonacci(n: int): int { writeln(fibonacci(10)); ``` -They can take a variable number of parameters: +Procedures can take a varying number of parameters. In this example the procedure `maxOf` takes two or more +parameters of the same type. This group of parameters is referred to as a *tuple* and is named `x` inside the +procedure. The number of elements `k` in this tuple is inferred from the number of parameters passed to the +procedure and is used to organize the calculations inside the procedure: ```chpl proc maxOf(x ...?k) { // take a tuple of one type with k elements @@ -41,6 +44,11 @@ proc maxOf(x ...?k) { // take a tuple of one type with k elements return maximum; } writeln(maxOf(1, -5, 123, 85, -17, 3)); +writeln(maxOf(1.12, 0.85, 2.35)); +``` +```output +123 +2.35 ``` Procedures can have default parameter values: @@ -61,6 +69,6 @@ parallelism, so we refer the interested readers to the official Chapel documenta ::::::::::::::::::::::::::::::::::::: keypoints - "Functions in Chapel are called procedures." - "Procedures can be recursive." -- "Procedures can take a variable number of parameters." +- "Procedures can take a varying number of parameters." - "Procedures can have default parameter values." :::::::::::::::::::::::::::::::::::::::::::::::: From 4654984f0796a90ccff5fd0f71a3fe7bfaeb9687 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 13:20:20 -0700 Subject: [PATCH 37/70] moved recursion to the end of the key points --- episodes/06-procedures.md | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/episodes/06-procedures.md b/episodes/06-procedures.md index 171fc95..52688cd 100644 --- a/episodes/06-procedures.md +++ b/episodes/06-procedures.md @@ -22,16 +22,35 @@ proc addOne(n) { // n is an.bash parameter writeln(addOne(10)); ``` -Procedures can be recursive: +Procedures can be recursive, as demonstrated below. In this example the procedure takes an integer number as a +parameter and returns an integer number -- more on this below. If the input parameter is 1 or 0, `fibonacci` +will return the same input parameter. If the input parameter is 2 or larger, `fibonacci` will call itself +recursively. ```chpl -proc fibonacci(n: int): int { +proc fibonacci(n: int): int { // input parameter type and procedure return type, respectively if n <= 1 then return n; return fibonacci(n-1) + fibonacci(n-2); } writeln(fibonacci(10)); ``` +The input parameter type `n: int` is enforced at compilation time. For example, if you try to pass a real-type +number to the procedure with `fibonacci(10.2)`, you will get an error "error: unresolved call". Similarly, the +return variable type is also enforced at compilation time. For example, replacing `return n` with `return 1.0` +in line 2 will result in "error: cannot initialize return value of type 'int(64)'". While specifying these +types might be optional (see the call out below), we highly recommend doing so in your code, as it will add +additional checks for your program. + +::::::::::::::::::::::::::::::::::::: callout + +If not specified, the procedure return type is inferred from the return variable type. This might not be +possible with a recursive procedure as the return type is the procedure type, and it is not known to the +compiler, so in this case (and in the `fibonacci` example above) we need to specify the procedure return type +explicitly. + +:::::::::::::::::::::::::::::::::::::::::::::::: + Procedures can take a varying number of parameters. In this example the procedure `maxOf` takes two or more parameters of the same type. This group of parameters is referred to as a *tuple* and is named `x` inside the procedure. The number of elements `k` in this tuple is inferred from the number of parameters passed to the @@ -51,7 +70,12 @@ writeln(maxOf(1.12, 0.85, 2.35)); 2.35 ``` -Procedures can have default parameter values: +Procedures can have default parameter values. If a parameter with the default value (like `y` in the example +below) is not passed to the procedure, it takes the default value inside the procedure. If it is passed with +another value, then this new value is used inside the procedure. + +In Chapel a procedure always returns a single value or a single data structure. In this example the procedure +returns a *tuple* (a structure) with two numbers inside, one integer and one real: ```chpl proc returnTuple(x: int, y: real = 3.1415926): (int,real) { @@ -68,7 +92,8 @@ parallelism, so we refer the interested readers to the official Chapel documenta ::::::::::::::::::::::::::::::::::::: keypoints - "Functions in Chapel are called procedures." -- "Procedures can be recursive." - "Procedures can take a varying number of parameters." +- "Optionally, you can specify input parameter types and the return variable type." - "Procedures can have default parameter values." +- "Procedures can be recursive. Recursive procedures require specifying the return variable type." :::::::::::::::::::::::::::::::::::::::::::::::: From 323ed660fdbdf78732d2115a97496387f0e33332 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 15:02:18 -0700 Subject: [PATCH 38/70] in Chapel 2.0 no longer need -o to have a similar-named executable; removing -o everywhere unless needed in that context --- episodes/01-intro.md | 6 +++--- episodes/02-variables.md | 4 ++-- episodes/03-ranges-arrays.md | 10 +++++----- episodes/04-conditionals.md | 6 +++--- episodes/05-loops.md | 10 +++++----- episodes/07-commandargs.md | 2 +- episodes/08-timing.md | 2 +- episodes/12-fire-forget-tasks.md | 10 +++++----- episodes/13-synchronization.md | 6 +++--- episodes/14-parallel-case-study.md | 4 ++-- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/episodes/01-intro.md b/episodes/01-intro.md index 4f92546..e34be0e 100644 --- a/episodes/01-intro.md +++ b/episodes/01-intro.md @@ -42,7 +42,7 @@ writeln('If we can see this, everything works!'); This program can then be compiled with the following bash command: ```bash -chpl --fast hello.chpl -o hello.o +chpl --fast hello.chpl ``` The flag `--fast` indicates the compiler to optimise the binary to run as fast as possible in the given @@ -81,14 +81,14 @@ salloc --time=0:30:0 --ntasks=1 --cpus-per-task=3 --mem-per-cpu=1000 --account=d and then inside that job compile and run the test code ```bash -chpl --fast hello.chpl -o hello.o +chpl --fast hello.chpl ./hello.o ``` For production jobs, you would compile the code and then submit a batch script to the queue: ```bash -chpl --fast hello.chpl -o hello.o +chpl --fast hello.chpl sbatch script.sh ``` diff --git a/episodes/02-variables.md b/episodes/02-variables.md index 76e72d5..7768e5f 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -30,7 +30,7 @@ writeln(4 ** 5); // exponentiation In this example, our code is called `operators.chpl`. You can compile it with the following commands: ```bash -chpl operators.chpl --fast -o operators.o +chpl operators.chpl --fast ./operators.o ``` @@ -109,7 +109,7 @@ writeln('The value of test is: ', test); writeln(test / 4); ``` ```bash -chpl variables.chpl -o variables.o +chpl variables.chpl ``` ```output The value of test is: 200 diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index d3d9be2..14c1e26 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -25,7 +25,7 @@ writeln('Our example range was set to: ', example_range); ``` ```bash -chpl ranges.chpl -o ranges.o +chpl ranges.chpl ./ranges.o ``` @@ -60,7 +60,7 @@ writeln('When set to a range: ', example_array); ``` ```bash -chpl ranges.chpl -o ranges.o +chpl ranges.chpl ./ranges.o ``` @@ -96,7 +96,7 @@ writeln(example_array); ``` ```bash -chpl ranges.chpl -o ranges.o +chpl ranges.chpl ./ranges.o ``` @@ -131,7 +131,7 @@ writeln(example_array); ``` ```bash -chpl ranges.chpl -o ranges.o +chpl ranges.chpl ./ranges.o ``` @@ -187,7 +187,7 @@ writeln('Temperature at start is: ', temp[x, y]); ``` ```bash -chpl base_solution.chpl -o base_solution +chpl base_solution.chpl ./base_solution ``` diff --git a/episodes/04-conditionals.md b/episodes/04-conditionals.md index 42c133e..10cb10d 100644 --- a/episodes/04-conditionals.md +++ b/episodes/04-conditionals.md @@ -27,7 +27,7 @@ writeln(1 <= 2); ``` ```bash -chpl conditionals.chpl -o conditionals.o +chpl conditionals.chpl ./conditionals.o ``` @@ -56,7 +56,7 @@ writeln(true || false); ``` ```bash -chpl conditionals.chpl -o conditionals.o +chpl conditionals.chpl ./conditionals.o ``` @@ -169,7 +169,7 @@ while (c < niter && delta >= tolerance) do ``` ```bash -chpl base_solution.chpl -o base_solution.o +chpl base_solution.chpl ./base_solution.o ``` diff --git a/episodes/05-loops.md b/episodes/05-loops.md index 0f0465c..4818a9d 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -80,7 +80,7 @@ statement. Now let's compile and execute our code again: ```bash -chpl base_solution.chpl -o base_solution +chpl base_solution.chpl ./base_solution ``` @@ -140,7 +140,7 @@ To see the evolution of the temperature at the top right corner of the plate, we `y`. This corner correspond to the first row (`x=1`) and the last column (`y=cols`) of the plate. ```bash -chpl base_solution.chpl -o base_solution +chpl base_solution.chpl ./base_solution ``` @@ -189,7 +189,7 @@ Note that 80 degrees is written as a real number 80.0. The division of integers then, as `rows` and `cols` are integers, we must have 80 as real so that the result is not truncated. ```bash -chpl base_solution.chpl -o base_solution +chpl base_solution.chpl ./base_solution ``` @@ -252,7 +252,7 @@ Clearly there is no need to keep the difference at every single position in the `delta` if we find a greater one. ```bash -chpl base_solution.chpl -o base_solution +chpl base_solution.chpl ./base_solution ``` @@ -291,7 +291,7 @@ writeln('The difference in temperatures between the last two iterations was: ',d and compile and execute our final code, ```bash -chpl base_solution.chpl -o base_solution +chpl base_solution.chpl ./base_solution ``` diff --git a/episodes/07-commandargs.md b/episodes/07-commandargs.md index 0126e1b..564fd5d 100644 --- a/episodes/07-commandargs.md +++ b/episodes/07-commandargs.md @@ -29,7 +29,7 @@ config const niter = 500; //number of iterations ``` ```bash -chpl base_solution.chpl -o base_solution +chpl base_solution.chpl ``` it can be initialised with a specific value, when executing the code at the command line, using the syntax: diff --git a/episodes/08-timing.md b/episodes/08-timing.md index cffde39..2ad11f5 100644 --- a/episodes/08-timing.md +++ b/episodes/08-timing.md @@ -81,7 +81,7 @@ writeln('The greatest difference in temperatures between the last two iterations ``` ```bash -chpl base_solution.chpl -o base_solution +chpl base_solution.chpl ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 ``` diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index 7c683dc..5c8c2fb 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -45,7 +45,7 @@ writeln('this is main thread, I am done...'); ``` ```bash -chpl begin_example.chpl -o begin_example +chpl begin_example.chpl ./begin_example ``` @@ -157,7 +157,7 @@ writeln("this message won't appear until all tasks are done..."); ``` ```bash -chpl cobegin_example.chpl -o cobegin_example +chpl cobegin_example.chpl ./cobegin_example ``` @@ -199,7 +199,7 @@ writeln("this message won't appear until all tasks are done..."); ``` ```bash -chpl coforall_example.chpl -o coforall_example +chpl coforall_example.chpl ./coforall_example --numoftasks=5 ``` @@ -251,7 +251,7 @@ writeln("this message won't appear until all tasks are done..."); ``` ```bash -chpl exercise_coforall.chpl -o exercise_coforall +chpl exercise_coforall.chpl ./exercise_coforall --numoftasks=5 ``` @@ -326,7 +326,7 @@ for i in 0..numoftasks-1 do ``` ```bash -chpl --fast exercise_coforall_2.chpl -o exercise_coforall_2 +chpl --fast exercise_coforall_2.chpl ./exercise_coforall_2 ``` diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index fc3f701..da9315c 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -49,7 +49,7 @@ writeln('this is main thread, I am done...'); ``` ```bash -chpl sync_example_1.chpl -o sync_example_1 +chpl sync_example_1.chpl ./sync_example_1 ``` @@ -154,7 +154,7 @@ writeln("and now it is done"); ``` ```bash -chpl sync_example_2.chpl -o sync_example_2 +chpl sync_example_2.chpl ./sync_example_2 ``` @@ -232,7 +232,7 @@ coforall id in 1..numtasks ``` ```bash -chpl atomic_example.chpl -o atomic_example +chpl atomic_example.chpl ./atomic_example ``` diff --git a/episodes/14-parallel-case-study.md b/episodes/14-parallel-case-study.md index 6c2c170..f6678e8 100644 --- a/episodes/14-parallel-case-study.md +++ b/episodes/14-parallel-case-study.md @@ -104,7 +104,7 @@ distribution for us. We will study data parallelism in the following lessons, bu benchmark solution with our `coforall` parallelization to see how the performance improved. ```bash -chpl --fast parallel_solution_1.chpl -o parallel1 +chpl --fast parallel1.chpl ./parallel1 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 ``` @@ -290,7 +290,7 @@ coforall taskid in 0..coltasks*rowtasks-1 do Using the solution in the Exercise 4, we can now compare the performance with the benchmark solution ```bash -chpl --fast parallel_solution_2.chpl -o parallel2 +chpl --fast parallel2.chpl ./parallel2 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 ``` From 9601fe9b49cafc9f24b0015db05901d7a7a08a17 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 15:06:40 -0700 Subject: [PATCH 39/70] merge two bash blocks; remove all prompt characters --- episodes/07-commandargs.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/episodes/07-commandargs.md b/episodes/07-commandargs.md index 564fd5d..73b7e63 100644 --- a/episodes/07-commandargs.md +++ b/episodes/07-commandargs.md @@ -28,13 +28,10 @@ this is to use **_config_** variables. When a variable is declared with the `con config const niter = 500; //number of iterations ``` -```bash -chpl base_solution.chpl -``` - it can be initialised with a specific value, when executing the code at the command line, using the syntax: ```bash +chpl base_solution.chpl ./base_solution --niter=3000 ``` From 818d1292220310d4c3d2ea8d03b2427114f3b05b Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 15:31:40 -0700 Subject: [PATCH 40/70] provide the solution to Challenge 4 --- episodes/07-commandargs.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/episodes/07-commandargs.md b/episodes/07-commandargs.md index 73b7e63..ccd1780 100644 --- a/episodes/07-commandargs.md +++ b/episodes/07-commandargs.md @@ -58,17 +58,32 @@ The greatest difference in temperatures between the last two iterations was: 0.0 ## Challenge 4: Can you do it? -Make `n`, `x`, `y`, `tolerance`, `rows` and `cols` configurable variables, and test the code simulating different +Make `outputFrequency`, `x`, `y`, `tolerance`, `rows` and `cols` configurable variables, and test the code +simulating different configurations. What can you conclude about the performance of the code? :::::::::::::::::::::::: solution -For example, lets use a 650 x 650 grid and observe the evolution of the temperature at the position (200,300) -for 10000 iterations or until the difference of temperature between iterations is less than 0.002; also, let's -print the temperature every 1000 iterations. +Let's prepend `config` to the following lines in our code: + +```chpl +config const rows = 100; // number of rows in the grid +config const cols = 100; // number of columns in the grid +config const niter = 10_000; // maximum number of iterations +config const x = 1; // row number for a printout +config const y = cols; // column number for a printout +config const tolerance: real = 0.0001; // smallest difference in temperature that would be accepted before stopping +config const outputFrequency: int = 20; // the temperature will be printed every outputFrequency iterations +``` + +We can then recompile the code and try modifying some of these parameters from the command line. For example, +let's use a 650 x 650 grid and observe the evolution of the temperature at the position (200,300) for 10,000 +iterations or until the difference of temperature between iterations is less than 0.002; also, let's print the +temperature every 1000 iterations. ```bash -./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 +chpl base_solution.chpl +./base_solution --rows=650 --cols=650 --x=200 --y=300 --tolerance=0.002 --outputFrequency=1000 ``` ```output From 611a716ad1c57d568bb5a42b66a0f8e504b64bdd Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 15:43:21 -0700 Subject: [PATCH 41/70] change the key point about measuring code performance --- episodes/08-timing.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/episodes/08-timing.md b/episodes/08-timing.md index 2ad11f5..17bd9e3 100644 --- a/episodes/08-timing.md +++ b/episodes/08-timing.md @@ -20,7 +20,7 @@ But first, we need a quantitative way to measure the performance of our code. T see how long it takes to finish a simulation. The UNIX command `time` could be used to this effect ```bash -time ./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 +time ./base_solution --rows=650 --cols=650 --x=200 --y=300 --tolerance=0.002 --outputFrequency=1000 ``` ```output @@ -58,11 +58,11 @@ achieved by modifying the code to output the information that we need. This proc An easy way to instrument our code with Chapel is by using the module `Time`. **_Modules_** in Chapel are libraries of useful functions and methods that can be used once the module is loaded. To load a module we use the keyword `use` followed by the name of the module. Once the Time module is loaded we can create a variable -of the type `Timer`, and use the methods `start`,`stop`and `elapsed` to instrument our code. +of the type `stopwatch`, and use the methods `start`,`stop`and `elapsed` to instrument our code. ```chpl use Time; -var watch: Timer; +var watch: stopwatch; watch.start(); //this is the main loop of the simulation @@ -82,7 +82,7 @@ writeln('The greatest difference in temperatures between the last two iterations ```bash chpl base_solution.chpl -./base_solution --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 +./base_solution --rows=650 --cols=650 --x=200 --y=300 --tolerance=0.002 --outputFrequency=1000 ``` ```output @@ -108,5 +108,5 @@ The greatest difference in temperatures between the last two iterations was: 0.0 ``` ::::::::::::::::::::::::::::::::::::: keypoints -- "Use UNIX `time` command or instrument your Chapel code to measure performance." +- "To measure performance, instrument your Chapel code using a stopwatch from the `Time` module." :::::::::::::::::::::::::::::::::::::::::::::::: From f8cabc62e256a3a1e70e38f2875b926cbd5ffa77 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 15:45:32 -0700 Subject: [PATCH 42/70] rename n outputFrequency in 14-parallel-case-study.md --- episodes/14-parallel-case-study.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/episodes/14-parallel-case-study.md b/episodes/14-parallel-case-study.md index f6678e8..19cbeae 100644 --- a/episodes/14-parallel-case-study.md +++ b/episodes/14-parallel-case-study.md @@ -105,7 +105,7 @@ benchmark solution with our `coforall` parallelization to see how the performanc ```bash chpl --fast parallel1.chpl -./parallel1 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 +./parallel1 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --outputFrequency=1000 ``` ```output @@ -291,7 +291,7 @@ Using the solution in the Exercise 4, we can now compare the performance with th ```bash chpl --fast parallel2.chpl -./parallel2 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --n=1000 +./parallel2 --rows=650 --cols=650 --x=200 --y=300 --niter=10000 --tolerance=0.002 --outputFrequency=1000 ``` ```output From 6cfa8259d43fa1c45f52d1f52a1abc31a6cc8448 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 15:55:15 -0700 Subject: [PATCH 43/70] finish sentences with a period --- episodes/11-parallel-intro.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/episodes/11-parallel-intro.md b/episodes/11-parallel-intro.md index f22bb80..97594ac 100644 --- a/episodes/11-parallel-intro.md +++ b/episodes/11-parallel-intro.md @@ -31,7 +31,7 @@ painters could executed the tasks **_truly in parallel_**. ::::::::::::::::::::::::::::::::::::: callout -Think of the CPU cores as the painters or workers that will execute your concurrent tasks +Think of the CPU cores as the painters or workers that will execute your concurrent tasks. :::::::::::::::::::::::::::::::::::::::::::::::: @@ -42,7 +42,7 @@ use the dispenser: One must wait while the other is being serviced. ::::::::::::::::::::::::::::::::::::: callout -Think of the shared memory in your computer as the central dispenser for all your workers +Think of the shared memory in your computer as the central dispenser for all your workers. :::::::::::::::::::::::::::::::::::::::::::::::: @@ -55,7 +55,7 @@ the paint of worker B and worker B must respond by sending the required colour. ::::::::::::::::::::::::::::::::::::: callout -Think of the memory distributed on each node of a cluster as the different dispensers for your workers +Think of the memory distributed on each node of a cluster as the different dispensers for your workers. :::::::::::::::::::::::::::::::::::::::::::::::: From b69eb96ad47d1606902f2c8efdec05b5e1e91b82 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 18:38:39 -0700 Subject: [PATCH 44/70] reformulate the analogy with multiple dispensers --- episodes/11-parallel-intro.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/episodes/11-parallel-intro.md b/episodes/11-parallel-intro.md index 97594ac..a0af2ea 100644 --- a/episodes/11-parallel-intro.md +++ b/episodes/11-parallel-intro.md @@ -27,7 +27,7 @@ can be executed simultaneously or in parallel. It all depends on the amount of r tasks. If there is only one painter, this guy could work for a while in one wall, then start painting another one, then work for a little bit on the third one, and so on. **_The tasks are being executed concurrently but not in parallel_**. If we have two painters for the job, then more parallelism can be introduced. Four -painters could executed the tasks **_truly in parallel_**. +painters could execute the tasks **_truly in parallel_**. ::::::::::::::::::::::::::::::::::::: callout @@ -55,7 +55,7 @@ the paint of worker B and worker B must respond by sending the required colour. ::::::::::::::::::::::::::::::::::::: callout -Think of the memory distributed on each node of a cluster as the different dispensers for your workers. +Think of the memory on each node of a cluster as a separate dispenser for your workers. :::::::::::::::::::::::::::::::::::::::::::::::: From fc5a9c09c79f65118caf34fb05fe044e58932a14 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 18:48:51 -0700 Subject: [PATCH 45/70] improve text around distributed memory --- episodes/11-parallel-intro.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/episodes/11-parallel-intro.md b/episodes/11-parallel-intro.md index a0af2ea..0019f97 100644 --- a/episodes/11-parallel-intro.md +++ b/episodes/11-parallel-intro.md @@ -66,7 +66,8 @@ executed completely independent from each other (no communications required). ## Parallel programming in Chapel Chapel provides high-level abstractions for parallel programming no matter the grain size of your tasks, -whether they run in a shared memory or a distributed memory environment, or whether they are executed +whether they run in a shared memory on one node or use memory distributed across multiple compute nodes, +or whether they are executed concurrently or truly in parallel. As a programmer you can focus in the algorithm: how to divide the problem into tasks that make sense in the context of the problem, and be sure that the high-level implementation will run on any hardware configuration. Then you could consider the details of the specific system you are going to @@ -87,10 +88,10 @@ In summary, we can have a set of several tasks; these tasks could be running: 3. in parallel by several processors distributed in different compute nodes, or 4. serially (one by one) by several processors distributed in different compute nodes. -Similarly, each of these tasks could be using variables located in: +Similarly, each of these tasks could be using variables -1. the local memory on the compute node where it is running, or -2. on distributed memory located in other compute nodes. +1. located in the local memory on the compute node where it is running, or +2. stored on other compute nodes. And again, Chapel could take care of all the stuff required to run our algorithm in most of the scenarios, but we can always add more specific detail to gain performance when targeting a particular scenario. From 3ec37d92e941aa48d351b20758cbd9d4249c3cb1 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 18:56:14 -0700 Subject: [PATCH 46/70] add a key point about Chapel communication under the hood --- episodes/11-parallel-intro.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/episodes/11-parallel-intro.md b/episodes/11-parallel-intro.md index 0019f97..faf43bb 100644 --- a/episodes/11-parallel-intro.md +++ b/episodes/11-parallel-intro.md @@ -99,4 +99,7 @@ we can always add more specific detail to gain performance when targeting a part ::::::::::::::::::::::::::::::::::::: keypoints - "Concurrency and locality are orthogonal concepts in Chapel: where the tasks are running may not be indicative of when they run, and you can control both in Chapel." +- "Problems with a lot of communication between tasks, or so called **_fine-grained_** parallel problems, are + typically more difficult to parallelize. As we will see later in these lessons, Chapel simplifies writing + **_fine-grained_** parallel codes by hiding a lot of communication complexity under the hood." :::::::::::::::::::::::::::::::::::::::::::::::: From f020ce671460578cd4e263b383a001eaaef06b83 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 19:00:37 -0700 Subject: [PATCH 47/70] fix typo --- episodes/12-fire-forget-tasks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index 5c8c2fb..98dd189 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -5,7 +5,7 @@ exercises: 30 --- :::::::::::::::::::::::::::::::::::::: questions -- "How do execute work in parallel?" +- "How do we execute work in parallel?" :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives From 29be289491e7db0675bbcb0822983af6fcb65f46 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 19:25:48 -0700 Subject: [PATCH 48/70] add lesson objectives to several chapters --- episodes/01-intro.md | 2 +- episodes/05-loops.md | 3 ++- episodes/07-commandargs.md | 2 +- episodes/08-timing.md | 2 +- episodes/11-parallel-intro.md | 2 +- episodes/12-fire-forget-tasks.md | 5 +++-- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/episodes/01-intro.md b/episodes/01-intro.md index e34be0e..81cc1c6 100644 --- a/episodes/01-intro.md +++ b/episodes/01-intro.md @@ -9,7 +9,7 @@ exercises: 15 :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives -- "Write and execute our first chapel program." +- "Write and execute our first Chapel program." :::::::::::::::::::::::::::::::::::::::::::::::: **_Chapel_** is a modern, open-source programming language that supports HPC via high-level diff --git a/episodes/05-loops.md b/episodes/05-loops.md index 4818a9d..dd323f5 100644 --- a/episodes/05-loops.md +++ b/episodes/05-loops.md @@ -9,7 +9,8 @@ exercises: 30 :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives -- "First objective." +- "Learn to use `for` loops to run over every element of an iterand." +- "Learn the difference between using `for` loops and using a `while` statement to repeatedly execute a code block. :::::::::::::::::::::::::::::::::::::::::::::::: To compute the new temperature, i.e. each element of `temp_new`, we need to add all the surrounding elements in diff --git a/episodes/07-commandargs.md b/episodes/07-commandargs.md index ccd1780..0d64589 100644 --- a/episodes/07-commandargs.md +++ b/episodes/07-commandargs.md @@ -9,7 +9,7 @@ exercises: 30 :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives -- "First objective." +- "Modifying code's constant parameters without re-compiling the code." :::::::::::::::::::::::::::::::::::::::::::::::: From the last run of our code, we can see that 500 iterations is not enough to get to a _steady state_ (a diff --git a/episodes/08-timing.md b/episodes/08-timing.md index 17bd9e3..d36e099 100644 --- a/episodes/08-timing.md +++ b/episodes/08-timing.md @@ -9,7 +9,7 @@ exercises: 30 :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives -- "First objective." +- "Measuring code performance by instrumenting the code." :::::::::::::::::::::::::::::::::::::::::::::::: The code generated after Exercise 4 is the basic implementation of our simulation. We will use it as a diff --git a/episodes/11-parallel-intro.md b/episodes/11-parallel-intro.md index faf43bb..7328368 100644 --- a/episodes/11-parallel-intro.md +++ b/episodes/11-parallel-intro.md @@ -9,7 +9,7 @@ exercises: 30 :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives -- "First objective." +- "Discuss some common concepts in parallel computing." :::::::::::::::::::::::::::::::::::::::::::::::: The basic concept of parallel computing is simple to understand: we divide our job into tasks that can be diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index 98dd189..e5f45d8 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -9,7 +9,8 @@ exercises: 30 :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives -- "First objective." +- "Launching multiple threads to execute tasks in parallel." +- "Learn how to use `begin`, `cobegin`, and `coforall` to spawn new tasks." :::::::::::::::::::::::::::::::::::::::::::::::: A Chapel program always start as a single main thread. You can then start concurrent tasks with the `begin` @@ -334,7 +335,7 @@ chpl --fast exercise_coforall_2.chpl the maximum value in x is: 1.0 ``` -We use the coforall to spawn tasks that work concurrently in a fraction of the array. The trick here is to +We use the `coforall` loop to spawn tasks that work concurrently in a fraction of the array. The trick here is to determine, based on the _taskid_, the initial and final indices that the task will use. Each task obtains the maximum in its fraction of the array, and finally, after the coforall is done, the main task obtains the maximum of the array from the maximums of all tasks. From 146bf4f8749a451f5672f95052bb72171fa6c7f9 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 19:50:28 -0700 Subject: [PATCH 49/70] reduce the number of loops in this standalone example to 10 so that output fits on the screen --- episodes/12-fire-forget-tasks.md | 42 +++++++++++++++----------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index e5f45d8..53c112f 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -18,27 +18,27 @@ statement. A task spawned by the `begin` statement will run in a different threa continues its normal execution. Consider the following example: ```chpl -var x=0; +var x = 0; writeln("This is the main thread starting first task"); begin { - var c=0; - while c<100 + var c = 0; + while c < 10 { - c+=1; - writeln('thread 1: ',x+c); + c += 1; + writeln('thread 1: ', x+c); } } writeln("This is the main thread starting second task"); begin { - var c=0; - while c<100 + var c = 0; + while c < 10 { - c+=1; - writeln('thread 2: ',x+c); + c += 1; + writeln('thread 2: ', x+c); } } @@ -54,6 +54,16 @@ chpl begin_example.chpl This is the main thread starting first task This is the main thread starting second task this is main thread, I am done... +thread 1: 1 +thread 1: 2 +thread 1: 3 +thread 1: 4 +thread 1: 5 +thread 1: 6 +thread 1: 7 +thread 1: 8 +thread 1: 9 +thread 1: 10 thread 2: 1 thread 2: 2 thread 2: 3 @@ -64,20 +74,6 @@ thread 2: 7 thread 2: 8 thread 2: 9 thread 2: 10 -thread 2: 11 -thread 2: 12 -thread 2: 13 -thread 1: 1 -thread 2: 14 -thread 1: 2 -thread 2: 15 -... -thread 2: 99 -thread 1: 97 -thread 2: 100 -thread 1: 98 -thread 1: 99 -thread 1: 100 ``` As you can see the order of the output is not what we would expected, and actually it is completely From abcd7f4b7d003568929710952cfa0b1027ac9a86 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Mon, 23 Sep 2024 21:40:29 -0700 Subject: [PATCH 50/70] convert the discussion into two multiple-choice exercises --- episodes/12-fire-forget-tasks.md | 95 ++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index 53c112f..18eec91 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -80,25 +80,98 @@ As you can see the order of the output is not what we would expected, and actual unpredictable. This is a well known effect of concurrent tasks accessing the same shared resource at the same time (in this case the screen); the system decides in which order the tasks could write to the screen. -::::::::::::::::::::::::::::::::::::::: discussion -## Discussion -- What would happen if in the last code we declare `c` in the main thread? -- What would happen if we try to modify the value of `x` inside a begin statement? -Discuss your observations. -::::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 1: what if `c` is defined globally? + +What would happen if in the last code we *move* the definition of `c` into the main thread, but try to assign +it from threads 1 and 2? Select one answer from these: + +1. The code will fail to compile. +1. The code will compile and run, but `c` will be updated by both threads at the same time (a *race + condition*), so that its final value will vary from one run to another. +1. The code will compile and run, and the two threads will be taking turns updating `c`, so that its final + value will always be the same. + +:::::::::::::::::::::::: solution + +We'll get an error at compilation ("cannot assign to const variable"), since then `c` would be defined within +the scope of the main thread, and we could modify its value only in the main thread. Any attempt to modify its +value inside threads 1 or 2 will produce a compilation error. + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: + + + + + + + +::::::::::::::::::::::::::::::::::::: challenge + +## Challenge 2: what if we have a second, local definition of `x`? + +What would happen if we try to insert a second definition `var x = 10;` inside the first `begin` statement? +Select one answer from these: + +1. The code will fail to compile. +1. The code will compile and run, and the inside the first `begin` statement the value `x = 10` will be used, + whereas inside the second `begin` statement the value `x = 0` will be used. +1. The new value `x = 10` will overwrite the global value `x = 0` in both threads 1 and 2. + +:::::::::::::::::::::::: solution + +The code will compile and run, and you will see the following output: + +```output +This is the main thread starting first task +This is the main thread starting second task +this is main thread, I am done... +thread 1: 11 +thread 1: 12 +thread 1: 13 +thread 1: 14 +thread 1: 15 +thread 1: 16 +thread 1: 17 +thread 1: 18 +thread 1: 19 +thread 1: 20 +thread 2: 1 +thread 2: 2 +thread 2: 3 +thread 2: 4 +thread 2: 5 +thread 2: 6 +thread 2: 7 +thread 2: 8 +thread 2: 9 +thread 2: 10 +``` + +::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: callout -All variables have a **_scope_** in which they can be used. The variables declared inside a concurrent tasks -are accessible only by the task. The variables declared in the main task can be read everywhere, but Chapel -won't allow two concurrent tasks to try to modify them. +All variables have a **_scope_** in which they can be used. The variables declared inside a concurrent task +are accessible only by that task. The variables declared in the main task can be read everywhere, but Chapel +won't allow other concurrent tasks to modify them. :::::::::::::::::::::::::::::::::::::::::::::::: + + + + + + ::::::::::::::::::::::::::::::::::::::: discussion ## Try this ... @@ -216,7 +289,7 @@ can be read by all tasks, while the variables declared inside, are available onl ::::::::::::::::::::::::::::::::::::: challenge -## Challenge 1: Can you do it? +## Challenge 3: Can you do it? Would it be possible to print all the messages in the right order? Modify the code in the last example as required. @@ -270,7 +343,7 @@ Note that `+` is a **_polymorphic_** operand in Chapel. In this case it concaten ::::::::::::::::::::::::::::::::::::: challenge -## Challenge 2: Can you do it? +## Challenge 4: Can you do it? Consider the following code: From 1b83e0e0da174e8d24ad9a7b9294e4fed7cc2f5f Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 08:05:51 -0700 Subject: [PATCH 51/70] add salloc command (as a refresher; covered earlier) to start an interactive job --- episodes/12-fire-forget-tasks.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index 18eec91..835ef92 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -13,6 +13,35 @@ exercises: 30 - "Learn how to use `begin`, `cobegin`, and `coforall` to spawn new tasks." :::::::::::::::::::::::::::::::::::::::::::::::: + + + + + +In the very first chapter where we showed how to run single-node Chapel codes. As a refresher, let's go over +this again. If you are running Chapel on your own computer, then you are all set, and you can simply compile +and run Chapel codes. If you are on a cluster, you will need to run Chapel codes inside interactive jobs. Here +so far we are covering only single-locale Chapel, so -- from the login node -- you can submit an interactive +job to the scheduler with a command like this one: + +```sh +salloc --time=2:0:0 --ntasks=1 --cpus-per-task=3 --mem-per-cpu=1000 +``` + +The details may vary depending on your cluster, e.g. different scheduler, requirement to specify an account or +reservation, etc, but the general idea remains the same: on a cluster you need to ask for resources before you +can run calculations. In this case we are asking for 2 hours maximum runtime, single MPI task (sufficient for +our parallelism in this chapter), 3 CPU cores inside that task, and 1000M maximum memory per core. The core +count means that we can run 3 threads in parallel, each on its own CPU core. Once your interactive job starts, +you can compile and run the Chapel codes below. Inside your Chapel code, when new threads start, they will be +able to utilize our 3 allocated CPU cores. + + + + + + + A Chapel program always start as a single main thread. You can then start concurrent tasks with the `begin` statement. A task spawned by the `begin` statement will run in a different thread while the main thread continues its normal execution. Consider the following example: From 7401f555fd7b2cb0fcfb7df8c96f5cd99171a6cb Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 08:48:28 -0700 Subject: [PATCH 52/70] include shells commands to check for CPU usage; remove the sentence asking them to resubmit their job with a different number of CPU cores --- episodes/12-fire-forget-tasks.md | 46 +++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index 835ef92..31710be 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -13,10 +13,7 @@ exercises: 30 - "Learn how to use `begin`, `cobegin`, and `coforall` to spawn new tasks." :::::::::::::::::::::::::::::::::::::::::::::::: - - - - +::::::::::::::::::::::::::::::::::::: callout In the very first chapter where we showed how to run single-node Chapel codes. As a refresher, let's go over this again. If you are running Chapel on your own computer, then you are all set, and you can simply compile @@ -36,11 +33,7 @@ count means that we can run 3 threads in parallel, each on its own CPU core. Onc you can compile and run the Chapel codes below. Inside your Chapel code, when new threads start, they will be able to utilize our 3 allocated CPU cores. - - - - - +:::::::::::::::::::::::::::::::::::::::::::::::: A Chapel program always start as a single main thread. You can then start concurrent tasks with the `begin` statement. A task spawned by the `begin` statement will run in a different thread while the main thread @@ -207,23 +200,44 @@ won't allow other concurrent tasks to modify them. Are the concurrent tasks, spawned by the last code, running truly in parallel? -The answer is: it depends on the number of cores available to your job. To verify this, let's modify the code -to get the tasks into an infinite loop. +The answer is: it depends on the number of cores available to your Chapel code. To verify this, let's modify the code +to get both threads 1 and 2 into an infinite loop: ```chpl begin { var c=0; - while c>-1 + while c > -1 { - c+=1; - //writeln('thread 1: ',x+c); + c += 1; + // the rest of the code in the thread } } ``` -Now submit your job asking for different amount of resources, and use system tools such as `top`or `ps` to -monitor the execution of the code. +Compile and run the code: + +```sh +chpl begin_example.chpl +./begin_example +``` + +If you are running this on your own computer, you can run `top` or `htop` or `ps` commands in another terminal +to check Chapel's CPU usage. If you are running inside an interactive job on a cluster, you can open a +different terminal, log in to the cluster, and open a bash shell on the node that is running your job (if your +cluster setup allows this): + +```sh +squeue -u $USER # check the jobID number +srun --jobid= --pty bash # put your jobID here +htop -u $USER -s PERCENT_CPU # display CPU usage and other information +``` + +In the output of `htop` you will see a table with the list of your processes, and in the "CPU%" column you +will see the percentage consumed by each process. Find the Chapel process, and if it shows that your CPU usage +is close to 300%, you are using 3 CPU cores. What do you see? + +Now exit `htop` by pressing *Q*. Also exit your interactive run by pressing *Ctrl-C*. ::::::::::::::::::::::::::::::::::::::::::::::::::: From b5ae56b1a796839c9819b551071cf050e4727940 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 09:17:49 -0700 Subject: [PATCH 53/70] update the code in the example; rewrite the sentence removing *polymorphic* --- episodes/12-fire-forget-tasks.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index 31710be..a1d8096 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -346,17 +346,16 @@ the array in order. The following code is a possible solution: ```chpl -var x=1; -config var numoftasks=2; +var x = 1; +config var numoftasks = 2; var messages: [1..numoftasks] string; -writeln("This is the main task: x = ",x); +writeln("This is the main task: x = ", x); -coforall taskid in 1..numoftasks do -{ - var c=taskid+1; - var s="this is task "+taskid+": x + "+taskid+" = "+(x+taskid)+". My value of c is: "+c; - messages[taskid]=s; +coforall taskid in 1..numoftasks do { + var c = taskid + 1; + messages[taskid] = 'this is task ' + taskid:string + + ': my value of c is ' + c:string + ' and x is ' + x:string; } for i in 1..numoftasks do writeln(messages[i]); @@ -378,8 +377,8 @@ this is task 5: x + 5 = 6. My value of c is: 6 this message won't appear until all tasks are done... ``` -Note that `+` is a **_polymorphic_** operand in Chapel. In this case it concatenates `strings` with `integers` -(which are transformed to strings). +Note that we need to convert integers to strings first (`taskid:string` converts `taskid` integer variable to +a string) before we can add them to other strings to form a message stored inside each `messages` element. ::::::::::::::::::::::::::::::::: :::::::::::::::::::::::::::::::::::::::::::::::: From 8a8ad543f00c6aec1667390a4e61be47a21d398a Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 10:51:59 -0700 Subject: [PATCH 54/70] fix the solution in Challenge 4; remoce the discussion on playing with the number of cores --- episodes/12-fire-forget-tasks.md | 66 +++++++++----------------------- 1 file changed, 19 insertions(+), 47 deletions(-) diff --git a/episodes/12-fire-forget-tasks.md b/episodes/12-fire-forget-tasks.md index a1d8096..ab2a28a 100644 --- a/episodes/12-fire-forget-tasks.md +++ b/episodes/12-fire-forget-tasks.md @@ -391,14 +391,14 @@ Consider the following code: ```chpl use Random; -config const nelem=5000000000; -var x: [1..n] int; +config const nelem = 100_000_000; +var x: [1..nelem] int; fillRandom(x); //fill array with random numbers -var mymax=0; +var mymax = 0; // here put your code to find mymax -writeln("the maximum value in x is: ",mymax); +writeln("the maximum value in x is: ", mymax); ``` Write a parallel code to find the maximum value in the array x. @@ -406,35 +406,22 @@ Write a parallel code to find the maximum value in the array x. :::::::::::::::::::::::: solution ```chpl -config const numoftasks=12; -const n=nelem/numoftasks; -const r=nelem-n*numoftasks; - -var d: [0..numoftasks-1] real; - -coforall taskid in 0..numoftasks-1 do -{ - var i: int; - var f: int; - if taskidd[taskid] then d[taskid]=x[c]; - } -} -for i in 0..numoftasks-1 do -{ - if d[i]>mymax then mymax=d[i]; +config const numtasks = 12; +const n = nelem/numtasks; // number of elements per thread +const r = nelem - n*numtasks; // these elements did not fit into the last thread + +var d: [1..numtasks] int; // local maxima for each thread + +coforall taskid in 1..numtasks do { + var i, f: int; + i = (taskid-1)*n + 1; + f = (taskid-1)*n + n; + if taskid == numtasks then f += r; // add r elements to the last thread + for j in i..f do + if x[j] > d[taskid] then d[taskid] = x[j]; } +for i in 1..numtasks do + if d[i] > mymax then mymax = d[i]; ``` ```bash @@ -456,21 +443,6 @@ maximum of the array from the maximums of all tasks. ::::::::::::::::::::::::::::::::::::::: discussion -## Discussion - -Run the code of last Exercise using different number of tasks, and different sizes of the array _x_ to see how -the execution time changes. For example: - -```bash -time ./exercise_coforall_2 --nelem=3000 --numoftasks=4 -``` - -Discuss your observations. Is there a limit on how fast the code could run? - -::::::::::::::::::::::::::::::::::::::::::::::::::: - -::::::::::::::::::::::::::::::::::::::: discussion - ## Try this ... Substitute the code to find _mymax_ in the last exercise with: From 1c17a40441cd8b94a2ea61b322770acdf8535a47 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 11:59:23 -0700 Subject: [PATCH 55/70] add objectives in 13-synchronization.md --- episodes/13-synchronization.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index da9315c..6bfdbad 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -9,7 +9,10 @@ exercises: 30 :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives -- "First objective." +- "Learn how to synchronize multiple threads using one of three mechanisms: `sync` statements, sync variables, + and atomic variables." +- "Learn that with shared memory access from multiple threads you can run into race conditions and deadlocks, + and learn how to recognize and solve these problems." :::::::::::::::::::::::::::::::::::::::::::::::: The keyword `sync` provides all sorts of mechanisms to synchronise tasks in Chapel. From 1ab70df3ca2ce01ac007262da1f53625c730cbb6 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 12:11:59 -0700 Subject: [PATCH 56/70] provide more description before the example with the sync statement (clarify all ther terms) --- episodes/13-synchronization.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index 6bfdbad..5192ae1 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -17,7 +17,12 @@ exercises: 30 The keyword `sync` provides all sorts of mechanisms to synchronise tasks in Chapel. -We can simply use `sync` to force the _parent_ task to stop and wait until its _spawned-child-task_ ends. +As we saw in the previous section, the `begin` statement will start a concurrent (or *child*) task that will +run in a different thread while the main (or *parent*) thread continues its normal execution. In this sense +the `begin` statement is non-blocking. If you want to pause the execution of the main thread and wait until +the child thread ends, you can prepend the `begin` statement with the `sync` statement. Consider the following +code; running this code, after the initial output line, you will first see all output from thread 1 and only +then the line "The first task is done..." and the rest of the output: ```chpl var x=0; From 0dd9f338e8b604db6a784d1c450b9f51ae9b0eb2 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 12:16:25 -0700 Subject: [PATCH 57/70] explain that *sync* can be either a statement or a type qualifier --- episodes/13-synchronization.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index 5192ae1..33b0bc0 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -15,7 +15,8 @@ exercises: 30 and learn how to recognize and solve these problems." :::::::::::::::::::::::::::::::::::::::::::::::: -The keyword `sync` provides all sorts of mechanisms to synchronise tasks in Chapel. +In Chapel the keyword `sync` can be either a statement or a type qualifier, providing two different +synchronization mechanisms for threads. Let's start with using `sync` as a statement. As we saw in the previous section, the `begin` statement will start a concurrent (or *child*) task that will run in a different thread while the main (or *parent*) thread continues its normal execution. In this sense From cbd7f252c9c6c0207cb890b7a2b28c424771188d Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 13:51:38 -0700 Subject: [PATCH 58/70] update the code for Chapel 2.x --- episodes/13-synchronization.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index 33b0bc0..bc77e10 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -153,12 +153,12 @@ var x: sync int, a: int; writeln("this is main task launching a new task"); begin { for i in 1..10 do writeln("this is new task working: ",i); - x = 2; + x.writeEF(2); writeln("New task finished"); } writeln("this is main task after launching new task... I will wait until it is done"); -a = x; // don't run this line until the variable x is written in the other task +a = x.readFF(); // don't run this line until the variable x is written in the other task writeln("and now it is done"); ``` @@ -194,6 +194,15 @@ statement? Discuss your observations. + + +abc + + + + + + ::::::::::::::::::::::::::::::::::::::::::::::::::: There are a number of methods defined for _sync_ variables. Suppose _x_ is a sync variable of a given type, From cadea23d6ded442e13492cee96d9b715bb28a570 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 14:48:24 -0700 Subject: [PATCH 59/70] explain the example with writeEF/readFF in more detail; change the discussion with an example that will block for sure, and provide the solution/explanation of why it blocks --- episodes/13-synchronization.md | 48 ++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index bc77e10..7b6b86a 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -148,17 +148,26 @@ To assign a new value to a _sync_ variable, its state must be _empty_ (after the completed, the state will be set as _full_). On the contrary, to read a value from a _sync_ variable, its state must be _full_ (after the read operation is completed, the state will be set as _empty_ again). +Starting from Chapel 2.x, you must use functions `writeEF` and `readFF` to perform blocking write and read +with sync variables. Below is an example to demonstrate the use of sync variables. Here we launch a new task +that is busy for a short time executing the loop. While this loop is running, the main task continues printing +the message "this is main task after launching new task... I will wait until it is done". As it takes time to +spawn a new thread, it is very likely that you will see this message before the output from the loop. Next, +the main task will attempt to read `x` and assign it to `a` which it can only do when `x` is full. We write +into `x` after the loop, so you will see the final message "and now it is done" only after the message "New +task finished". In other words, reading `x`, we pause the execution of the main thread. + ```chpl var x: sync int, a: int; writeln("this is main task launching a new task"); begin { for i in 1..10 do writeln("this is new task working: ",i); - x.writeEF(2); + x.writeEF(2); // assign 2 to x writeln("New task finished"); } -writeln("this is main task after launching new task... I will wait until it is done"); -a = x.readFF(); // don't run this line until the variable x is written in the other task +writeln("this is main task after launching new task... I will wait until it is done"); +a = x.readFE(); // don't run this line until the variable x is written in the other task writeln("and now it is done"); ``` @@ -169,7 +178,7 @@ chpl sync_example_2.chpl ```output this is main task launching a new task -this is main task after launching new task... I will wait until it is done +this is main task after launching new task... I will wait until it is done this is new task working: 1 this is new task working: 2 this is new task working: 3 @@ -188,21 +197,28 @@ and now it is done ## Discussion -What would happen if we assign a value to _x_ right before launching the new task? What would happen if we -assign a value to _x_ right before launching the new task and after the _writeln("and now it is done");_ -statement? - -Discuss your observations. - - - -abc - - - +What would happen if we try to read `x` inside the new task as well, i.e. we have the following `begin` +statement, without changing the rest of the code: +```chpl +begin { + for i in 1..10 do writeln("this is new task working: ",i); + x.writeEF(2); + writeln("New task finished"); + x.readFE(); +} +``` +:::::::::::::::::::::::: solution +The code will block (run forever), and you would need to press *Ctrl-C* to halt its execution. In this example +we try to read `x` in two places: the main task and the new task. When we read a sync variable with `readFE`, +the state of the sync variable is set to empty when this method completes. In other words, one of the two +`readFE` calls will succeed (which one -- depends on the runtime) and will mark the variable as empty. The +other `readFE` will then attempt to read it but it will block waiting for `x` to become full again (which will +never happen). In the end, the execution of either the main thread or the child thread will block, hanging the +entire code. +::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::::::::::::::::: There are a number of methods defined for _sync_ variables. Suppose _x_ is a sync variable of a given type, From 7b29ed495fa95f9ff97a61490d679adf49b8641f Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 14:56:19 -0700 Subject: [PATCH 60/70] correct style --- episodes/13-synchronization.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/episodes/13-synchronization.md b/episodes/13-synchronization.md index 7b6b86a..74f10aa 100644 --- a/episodes/13-synchronization.md +++ b/episodes/13-synchronization.md @@ -221,10 +221,11 @@ entire code. ::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::::::::::::::::: -There are a number of methods defined for _sync_ variables. Suppose _x_ is a sync variable of a given type, +There are a number of methods defined for _sync_ variables. If `x` is a sync variable of a given type, you can +use the following functions: ```chpl -// general methods +// non-blocking methods x.reset() //will set the state as empty and the value as the default of x's type x.isfull() //will return true is the state of x is full, false if it is empty From 21db06a0a81cac6082c872a387fd663fec377e01 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 15:16:57 -0700 Subject: [PATCH 61/70] reformulate the key point about all variables having a type --- episodes/02-variables.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/episodes/02-variables.md b/episodes/02-variables.md index 7768e5f..c0da915 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -229,6 +229,7 @@ const outputFrequency: int = 20; // the temperature will be printed every outp ::::::::::::::::::::::::::::::::::::: keypoints - "A comment is preceded with `//` or surrounded by `/* and `*/`" -- "All variables hold a certain type of data." +- "All variables in Chapel have a type, whether assigned explicitly by the user, or chosen by the Chapel + compiler based on its value." - "Reassigning a new value to a `const` variable will produce an error during compilation. If you want to assign a new value to a variable, declare that variable with the `var` keyword." :::::::::::::::::::::::::::::::::::::::::::::::: From b5b6d205f0ba009d1303f558a76ae9eeb9058b17 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 15:28:41 -0700 Subject: [PATCH 62/70] break procedure definitions and usage into separate code blocks --- episodes/06-procedures.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/episodes/06-procedures.md b/episodes/06-procedures.md index 52688cd..a6cc177 100644 --- a/episodes/06-procedures.md +++ b/episodes/06-procedures.md @@ -16,9 +16,14 @@ Similar to other programming languages, Chapel lets you define your own function 'procedures' in Chapel and have an easy-to-understand syntax: ```chpl -proc addOne(n) { // n is an.bash parameter +proc addOne(n) { // n is an input parameter return n + 1; } +``` + +To call this procedure, you would use its name: + +```chpl writeln(addOne(10)); ``` @@ -32,6 +37,8 @@ proc fibonacci(n: int): int { // input parameter type and procedure return type, if n <= 1 then return n; return fibonacci(n-1) + fibonacci(n-2); } +``` +```chpl writeln(fibonacci(10)); ``` @@ -62,6 +69,8 @@ proc maxOf(x ...?k) { // take a tuple of one type with k elements for i in 2..k do maximum = if maximum < x[i] then x[i] else maximum; return maximum; } +``` +```chpl writeln(maxOf(1, -5, 123, 85, -17, 3)); writeln(maxOf(1.12, 0.85, 2.35)); ``` @@ -81,6 +90,8 @@ returns a *tuple* (a structure) with two numbers inside, one integer and one rea proc returnTuple(x: int, y: real = 3.1415926): (int,real) { return (x,y); } +``` +```chpl writeln(returnTuple(1)); writeln(returnTuple(x=2)); writeln(returnTuple(x=-10, y=10)); From bc91fe26265fedb3e59d94ba3c9c9def355fd096 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 15:44:46 -0700 Subject: [PATCH 63/70] use dashes for itemized lists; restore original indentation in **No additional restrictions** --- LICENSE.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 2e20476..6ac7dd3 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -29,16 +29,16 @@ Under the following terms: any reasonable manner, but not in any way that suggests the licensor endorses you or your use. -- **No additional restrictions**---You may not apply legal terms or - technological measures that legally restrict others from doing anything the - license permits. With the understanding that: +**No additional restrictions**---You may not apply legal terms or +technological measures that legally restrict others from doing anything the +license permits. With the understanding that: Notices: -* You do not have to comply with the license for elements of the material in +- You do not have to comply with the license for elements of the material in the public domain or where your use is permitted by an applicable exception or limitation. -* No warranties are given. The license may not give you all of the permissions +- No warranties are given. The license may not give you all of the permissions necessary for your intended use. For example, other rights such as publicity, privacy, or moral rights may limit how you use the material. From 6a41eb36f4d651ec2de8d0ceef65dc8ae53c6a40 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 16:09:40 -0700 Subject: [PATCH 64/70] switch to using dashes for itemized lists through all chapters --- episodes/01-intro.md | 8 ++++---- episodes/02-variables.md | 14 +++++++------- episodes/22-domains.md | 12 ++++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/episodes/01-intro.md b/episodes/01-intro.md index 81cc1c6..f2a74c7 100644 --- a/episodes/01-intro.md +++ b/episodes/01-intro.md @@ -120,13 +120,13 @@ at the past iteration (or the initial conditions in case we are at the first ite So, our objective is to: > ## Goals -> 1. Write a code to implement the difference equation above.The code should +> 1. Write a code to implement the difference equation above. The code should > have the following requirements: > -> * It should work for any given number of rows and columns in the grid. -> * It should run for a given number of iterations, or until the difference +> - It should work for any given number of rows and columns in the grid. +> - It should run for a given number of iterations, or until the difference > between `temp_new` and `temp` is smaller than a given tolerance value. -> * It should output the temperature at a desired position on the grid every +> - It should output the temperature at a desired position on the grid every > given number of iterations. > > 2. Use task parallelism to improve the performance of the code and run it in diff --git a/episodes/02-variables.md b/episodes/02-variables.md index c0da915..2ebaf08 100644 --- a/episodes/02-variables.md +++ b/episodes/02-variables.md @@ -89,14 +89,14 @@ it. Although we already kind of know why the error was caused (we tried to reass variable, which by definition cannot be changed), let's walk through the error as an example of how to troubleshoot our programs. -* `variables.chpl:2:` indicates that the error was caused on line 2 of our `variables.chpl` file. +- `variables.chpl:2:` indicates that the error was caused on line 2 of our `variables.chpl` file. -* `error:` indicates that the issue was an error, and blocks compilation. Sometimes the compiler will just +- `error:` indicates that the issue was an error, and blocks compilation. Sometimes the compiler will just give us warning or information, not necessarily errors. When we see something that is not an error, we should carefully read the output and consider if it necessitates changing our code. Errors must be fixed, as they will block the code from compiling. -* `cannot assign to const variable` indicates that we were trying to reassign a `const` variable, which is +- `cannot assign to const variable` indicates that we were trying to reassign a `const` variable, which is explicitly not allowed in Chapel. To fix this error, we can change `const` to `var` when declaring our `test` variable. `var` indicates a @@ -123,10 +123,10 @@ The value of test is: 200 In Chapel, to initialize a variable we must specify the type of the variable, or initialise it in place with some value. The common variable types in Chapel are: -* integer `int` (positive or negative whole numbers) -* floating-point number `real` (decimal values) -* Boolean `bool` (true or false) -* string `string` (any type of text) +- integer `int` (positive or negative whole numbers) +- floating-point number `real` (decimal values) +- Boolean `bool` (true or false) +- string `string` (any type of text) These two variables below are initialized with the type. If no initial value is given, Chapel will initialise a variable with a default value depending on the declared type, for example 0 for integers and 0.0 for real diff --git a/episodes/22-domains.md b/episodes/22-domains.md index e8c59c1..d6ee619 100644 --- a/episodes/22-domains.md +++ b/episodes/22-domains.md @@ -508,10 +508,10 @@ Now total energy should be conserved, as nothing leaves the domain. Let us write the final solution to disk. There are several caveats: -* works only with ASCII -* Chapel can also write binary data but nothing can read it (checked: not the +- works only with ASCII +- Chapel can also write binary data but nothing can read it (checked: not the endians problem!) -* would love to write NetCDF and HDF5, probably can do this by calling C/C++ +- would love to write NetCDF and HDF5, probably can do this by calling C/C++ functions from Chapel We'll add the following to our code to write ASCII: @@ -528,9 +528,9 @@ Run the code and check the file *output.dat*: it should contain the array T afte - - - + + + ::::::::::::::::::::::::::::::::::::: keypoints From 23b7c07db6fef317a05373f498b87d392428c7a5 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Tue, 24 Sep 2024 16:10:32 -0700 Subject: [PATCH 65/70] use temp variable to write the solution to disk --- episodes/22-domains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/episodes/22-domains.md b/episodes/22-domains.md index d6ee619..6d08c80 100644 --- a/episodes/22-domains.md +++ b/episodes/22-domains.md @@ -520,7 +520,7 @@ We'll add the following to our code to write ASCII: use IO; var myFile = open("output.dat", iomode.cw); // open the file for writing var myWritingChannel = myFile.writer(); // create a writing channel starting at file offset 0 -myWritingChannel.write(T); // write the array +myWritingChannel.write(temp); // write the array myWritingChannel.close(); // close the channel ``` From bd3d7e63da0691a367428e9be07b2532e5b510b5 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 27 Sep 2024 08:24:58 -0700 Subject: [PATCH 66/70] move first mention of arrays down the text --- episodes/03-ranges-arrays.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index 14c1e26..7329f6a 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -15,9 +15,7 @@ exercises: 30 ## Ranges and Arrays A series of integers (1,2,3,4,5, for example), is called a **_range_**. Ranges are generated with the `..` -operator, and are useful, among other things, to declare **_arrays_** of variables. - -Let's examine what a range looks like (`ranges.chpl` in this example): +operator. Let's examine what a range looks like (`ranges.chpl` in this example): ```chpl var example_range = 0..10; @@ -33,10 +31,9 @@ chpl ranges.chpl Our example range was set to: 0..10 ``` -An array is a multidimensional collection of values of the same type. Arrays can be any size, and are defined -using ranges. Let's -define a 1-dimensional array of the size `example_range` and see what it looks like. Notice how the size of an -array is included with its type. +Among other uses, ranges can be used to declare **_arrays_** of variables. An array is a multidimensional +collection of values of the same type. Arrays can be of any size. Let's define a 1-dimensional array of the +size `example_range` and see what it looks like. Notice how the size of an array is included with its type. ```chpl var example_range = 0..10; From c13a37ef9a19190b8d4948937896a48d9bbc2d66 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 27 Sep 2024 11:06:58 -0700 Subject: [PATCH 67/70] introduce a very simple loop to show range usage --- episodes/03-ranges-arrays.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index 7329f6a..11cd77f 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -15,11 +15,14 @@ exercises: 30 ## Ranges and Arrays A series of integers (1,2,3,4,5, for example), is called a **_range_**. Ranges are generated with the `..` -operator. Let's examine what a range looks like (`ranges.chpl` in this example): +operator. Let's examine what a range looks like; we store the following code as `ranges.chpl`. Here we +introduce a very simple loop, cycling through all elements of the range and printing their values (we will +study `for` loops in a separate section): ```chpl var example_range = 0..10; writeln('Our example range was set to: ', example_range); +for x in example_range do writeln(x); ``` ```bash @@ -29,6 +32,11 @@ chpl ranges.chpl ```output Our example range was set to: 0..10 +0 +1 +... +9 +10 ``` Among other uses, ranges can be used to declare **_arrays_** of variables. An array is a multidimensional From 85c1a07f3cbb211ab5c12fc7e512644bd8696458 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 27 Sep 2024 11:10:01 -0700 Subject: [PATCH 68/70] remove *One final thing* --- episodes/03-ranges-arrays.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index 11cd77f..c12e7b0 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -81,8 +81,8 @@ different from languages like Python where this does not happen. ## Indexing elements -One final thing - we can retrieve and reset specific values of an array using `[]` notation. Let's try -retrieving and setting a specific value in our example so far: +We can retrieve and reset specific values of an array using `[]` notation. Let's try retrieving and setting a +specific value in our example so far: ```chpl var example_range = 0..10; From 1e1b70adebd5ace18c10d0ec3601584ff588cd05 Mon Sep 17 00:00:00 2001 From: Alex Razoumov Date: Fri, 27 Sep 2024 11:45:50 -0700 Subject: [PATCH 69/70] add a sentence about the dual use of square brackets with arrays: declaration and subsetting --- episodes/03-ranges-arrays.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/episodes/03-ranges-arrays.md b/episodes/03-ranges-arrays.md index c12e7b0..7bf2bd4 100644 --- a/episodes/03-ranges-arrays.md +++ b/episodes/03-ranges-arrays.md @@ -81,8 +81,10 @@ different from languages like Python where this does not happen. ## Indexing elements -We can retrieve and reset specific values of an array using `[]` notation. Let's try retrieving and setting a -specific value in our example so far: +We can retrieve and reset specific values of an array using `[]` notation. Note that we use the same square +bracket notation in two different contexts: (1) to declare an array, with the square brackets containing the +array's full index range `[example_range]`, and (2) to access specific array elements, as we will see +below. Let's try retrieving and setting a specific value in our example so far: ```chpl var example_range = 0..10; From ab353822ff77e0ff52a9462afb0d9d5a4fd9a056 Mon Sep 17 00:00:00 2001 From: Trevor Keller Date: Wed, 2 Oct 2024 13:56:39 -0400 Subject: [PATCH 70/70] Updated human-readable license text. --- LICENSE.md | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 6ac7dd3..1c082b0 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -7,47 +7,50 @@ title: "Licenses" All High Performance Computing Carpentry instructional material is made available under the [Creative Commons Attribution license][cc-by-human]. The following is a human-readable summary of -(and not a substitute for) the [full legal text of the CC BY 4.0 +(and not a substitute for) the full legal text of the [CC BY 4.0 license][cc-by-legal]. -You are free: +### You are free to -- to **Share**---copy and redistribute the material in any medium or format -- to **Adapt**---remix, transform, and build upon the material +**Share**---copy and redistribute the material in any medium or format for any +purpose, even commercially. -for any purpose, even commercially. +**Adapt**---remix, transform, and build upon the material for any purpose, even +commercially. The licensor cannot revoke these freedoms as long as you follow the license terms. -Under the following terms: +### Under the following terms -- **Attribution**---You must give appropriate credit (mentioning that your work - is derived from work that is Copyright (c) High Performance Computing Carpentry and, where - practical, linking to ), provide a [link to the - license][cc-by-human], and indicate if changes were made. You may do so in - any reasonable manner, but not in any way that suggests the licensor endorses - you or your use. +**Attribution**---You must give appropriate credit, provide a [link to the +license][cc-by-human], and indicate if changes were made. You may do so in any +reasonable manner, but not in any way that suggests the licensor endorses you +or your use. -**No additional restrictions**---You may not apply legal terms or -technological measures that legally restrict others from doing anything the -license permits. With the understanding that: +**No additional restrictions**---You may not apply legal terms or technological +measures that legally restrict others from doing anything the license permits. -Notices: +### Notices -- You do not have to comply with the license for elements of the material in - the public domain or where your use is permitted by an applicable exception - or limitation. -- No warranties are given. The license may not give you all of the permissions - necessary for your intended use. For example, other rights such as publicity, - privacy, or moral rights may limit how you use the material. +You do not have to comply with the license for elements of the material in the +public domain or where your use is permitted by an applicable exception or +limitation. + +No warranties are given. The license may not give you all of the permissions +necessary for your intended use. For example, other rights such as publicity, +privacy, or moral rights may limit how you use the material. ## Software Except where otherwise noted, the example programs and other software provided -by High Performance Computing Carpentry are made available under the [OSI][osi]-approved [MIT +by HPC Carpentry are made available under the [OSI][osi]-approved [MIT license][mit-license]. +### MIT License + +Copyright © 2024 HPC Carpentry + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to