Merge pull request #370 from cumulus-nasa/develop

deploy latest

.circleci/config.yml

@@ -0,0 +1,47 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: node:6.9
+    working_directory: ~/cumulus-dashboard
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - cumulus-dashboard-{{ .Branch }}-{{ checksum "yarn.lock" }}
+            - cumulus-dashboard-{{ .Branch }}
+            - cumulus-dashboard-develop
+            - cumulus-dashboard
+      - run:
+          name: Install dependencies
+          command: |
+            apt-get update && apt-get install -y python-dev python-pip apt-transport-https
+            curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+            echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+            apt-get update && apt-get install -y yarn
+            pip install --upgrade --user awscli
+            yarn install --cache-folder ~/cumulus-dashboard/yarn-cache
+
+      - run:
+          name: Run tests
+          command: yarn test
+      - deploy:
+          name: Deploy develop branch to the dev CloudFormation stack
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "develop" ]; then
+              yarn run staging
+              ~/.local/bin/aws s3 sync ~/cumulus-dashboard/dist/. s3://cumulus-dashboard --acl=public-read
+            fi
+      - deploy:
+          name: Deploy master branch to the prod CloudFormation stack
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "master" ]; then
+              yarn run production
+              ~/.local/bin/aws s3 sync ~/cumulus-dashboard/dist/. s3://cumulus-dashboard-prod --acl=public-read
+            fi
+
+      - save_cache:
+          key: cumulus-dashboard-{{ .Branch }}-{{ checksum "yarn.lock" }}
+          paths:
+            - ~/cumulus-dashboard/node_modules
+            - ~/cumulus-dashboard/yarn-cache

.gitignore

@@ -4,4 +4,6 @@ node_modules
 .DS_Store
 *.swp
 dist
-app/scripts/config/local.js
+app/scripts/config/local.js
+gulp-cache
+/artifacts

README.md

@@ -2,6 +2,11 @@
 
 Code to generate and deploy the dashboard for the Cumulus API.
 
+## Documentation
+
+- [Usage](https://github.com/cumulus-nasa/cumulus-dashboard/blob/develop/USAGE.md)
+- [Technical documentation on tables](https://github.com/cumulus-nasa/cumulus-dashboard/blob/develop/TABLES.md)
+
 ## Wireframes and mocks
 
 - [Designs](https://www.dropbox.com/sh/zotoy2nuozizufz/AAAiOpbAv2Gp0BU-HIu5aILra?dl=0)
@@ -19,6 +24,51 @@ npm install
 npm run serve
 ```
 
+## Deploying in Bamboo
+
+### Build modules package
+
+```(bash)
+mkdir -p artifacts
+docker run \
+  -e RELEASE_UID=$(id -u) \
+  -e RELEASE_GID=$(id -g) \
+  --rm \
+  -v "$(pwd):/source:ro" \
+  -v "$(pwd)/artifacts:/artifacts" \
+  node \
+  /source/ngap/bamboo/build_modules_package.sh
+```
+
+### Run tests
+
+```(bash)
+tar -xf modules.tar
+mkdir -p artifacts
+docker run \
+  -e RELEASE_UID=$(id -u) \
+  -e RELEASE_GID=$(id -g) \
+  --rm \
+  -v "$(pwd):/source:ro" \
+  -v "$(pwd)/artifacts:/artifacts" \
+  node \
+  /source/ngap/bamboo/run_tests.sh
+```
+
+### Build release package
+
+```(bash)
+mkdir -p artifacts
+docker run \
+  -e RELEASE_UID=$(id -u) \
+  -e RELEASE_GID=$(id -g) \
+  --rm \
+  -v "$(pwd):/source:ro" \
+  -v "$(pwd)/artifacts:/artifacts" \
+  node \
+  /source/ngap/bamboo/build_release_package.sh
+```
+
 ## Adding a new page
 
 ### 1. Create the component

TABLES.md

@@ -0,0 +1,46 @@
+# Table documentation
+
+A lot of logic is encapsulated in the table components. They are central abstractions and wrap functionality like sorting, pagination, selection, and search. The main components are `sortable-table` and `list-view`.
+
+## `sortable-table`
+
+A basic table component that supports row selection and dumb sorting (see below).
+
+**Props**
+
+- **primaryIdx**: Column number to apply bold typeface to. Default is `0` or far-left column.
+- **data**: Array of data items. Items can be objects or arrays, depending on the accessor functions defined in `row`.
+- **header**: Array of strings representing the header row.
+- **row**: Array of items representing columns in each row. Items can be accessor functions with the arguments `data[k], k, data` (where `k` is the index of the current loop), or string values, ie `"collectionName"`.
+- **props**: Array of property names to send to elasticsearch for a re-ordering query.
+- **sortIdx**: The current index of the `props` array to sort by.
+- **order**: Either 'desc' or 'asc', corresponding to sort order.
+- **changeSortProps**: Callback when a new sort order is defined, passed an object with the properties `{ sortIdx, order }`.
+- **onSelect**: Callback when a row is selected (or unselected), passed a string id corresponding to the `rowId` selector.
+- **canSelect**: Boolean value defining whether 1. rows can be selected and 2. to render check marks.
+- **selectedRows**: Array of row ID's corresponding to rows that are currently selected.
+- **rowId**: String accessor to define as that row's id, ie `collectionName`.
+
+Note, `props`, `sortIdx`, `order`, and `changeSortProps` only apply to components that implement smart searching, such as `list-view`. This base component does internal prop checking to determine whether it uses smart or dumb sorting, based on whether the above props are defined.
+
+## `list-view`
+
+Wraps `sortable-table` and implements auto-update and smart sort. When a new sort order is requested, `sortable-table` calls a callback in `list-view`, which in turn dispatches the new sort parameters to the redux store. When `list-view` detects the new query in `componentWillReceiveProps`, it cancels the previous auto-update interval and creates a new one with the updated query parameters.
+
+**Props**
+
+- **list**: Parent data structure, ie `state.granules.list` or `state.collections.list`. Expected to contain `{ data, inflight, error, meta }` properties corresponding to all `list` state objects.
+- **dispatch**: Redux dispatch function.
+- **action**: Redux-style action to send, ie `listCollections`.
+- **tableHeader**: Corresponds to `sortableTable#header`.
+- **tableRow**: Corresponds to `sortableTable#row`.
+- **tableSortProps**: Corresponds to `sortableTable#props`.
+- **sortIdx**: Corresponds to `sortableTable#sortIdx`.
+- **query**: Array of configuration objects passed to `forms/batch-async-command`.
+- **rowId** Corresponds to `sortableTable#rowId`.
+
+## Dumb vs smart sort
+
+Dumb sorting uses `Array.sort()` within the component to re-order table data that has **already** been received from the API. Smart sorting initiates a new API request, passing the sort parameter to the server (elasticsearch) which returns a sorted response.
+
+Dumb sorting is for smaller, simple tables that do not need pagination.

USAGE.md

@@ -0,0 +1,55 @@
+# Viewing data
+
+Cumulus Dashboard shows comprehensive data on the status and workings of the Cumulus system. In this Dashboard you will find both aggregate and individual data slices.
+
+Cumulus is a moving system. Almost all of the data views in Dashboard will update automatically to show the latest data every 15 seconds, so you don't have to refresh the page manually. Many pages provide a timer and adjacent clickable span to stop automatic updates, should you need to. Once stopped, you can click again to restart automatic updates.
+
+## Searching and filtering
+
+Many tables provide dropdowns to filter the data based on attributes like status or parent collection. Often, you will find search fields where you can enter arbitrary search strings. As you type, the Cumulus Dashboard and API will attempt to find records containing attributes with **prefixes** that match your search string.
+
+For example, say a granule belongs to a PDR with the name `MODAPSops8.15810456.PDR`. In the granules section under "All Granules," searching for "MODAPS" will return a selection containing this granule. However, searching for "1581" will not return this granule.
+
+## Sorting
+
+In the data tables, small up and down arrows next to the table header means you can click re-order the table based on that column. Clicking the same header again will change the order from ascending to descending, and vice versa.
+
+Some columns cannot be sorted, and these will not show small arrows.
+
+# Performing actions on data
+
+Depending on the type, some sections will allow you to modify the data using buttons. These actions range from reingesting to reprocessing and deleting.
+
+Some actions cannot be performed on all records. For example, you can't delete a granule without removing it from CMR first. In general, these actions will be grayed out. However, if they are not grayed out but not allowed, the Dashboard will show an error when you try to perform the action.
+
+Actions on individual records are performed as soon as you click the button. On bulk records, a notification will pop up asking you to confirm your action, then show a loading indicator that will close automatically when all actions are successfully sent to Cumulus API.
+
+# Adding and editing records
+
+In the Cumulus Dashboard you can create and edit existing collections and providers. Links to create new records are at the top of the collections and providers sections, and links to edit an existing records are in the detail pages for single collections and providers.
+
+Whether you're editing or creating a new record, the form will be the same. This form corresponds to a data schema that Cumulus API defines, and which the Dashboard queries. This means the form you see in the Dashboard should always correspond with the schema that the API uses internally.
+
+## Required items
+
+Items marked with an asterisk `*` are required. If you omit a required field and try to submit the form, it will fail without contacting the API.
+
+The displayed error message will then tell you which required fields you need to fill out before you can submit.
+
+## Validation
+
+Once you successfully submit a form, the API will do it's own validation to confirm that the data you submitted matches the format that it expects. Some ways it could return a failed notification is, for example, you try to create a collection using a collection name that already exists in the database.
+
+For these, you should also see a corresponding error message at the top of the page.
+
+## Editing existing records
+
+Editing existing records is in practice almost the same as creating a new record, except that the form you see will be pre-filled with data from that record.
+
+## Fixing a mistake
+
+The forms do not yet support undo or redo actions, so if you accidentally make a modification that you cannot manually revert, refreshing the page will reset the form to the last saved state.
+
+## On successful save
+
+When you successfully save a record, you will be redirected to the overview for that section. If you do not see your changes immediately represented in the tables, check back in 15 or 30 seconds, as it sometimes takes a bit for the changes to propagate through the Cumulus system.

app/scripts/actions/index.js

@@ -253,7 +253,7 @@ export const getResources = (options) => wrapRequest(null, get, {
 
 // count queries *must* include type and field properties.
 export const getCount = (options) => wrapRequest(null, get, {
-  url: url.resolve(root, 'stats/count'),
+  url: url.resolve(root, 'stats/aggregate'),
   qs: Object.assign({ type: 'must-include-type', field: 'status' }, options)
 }, COUNT);
 

app/scripts/components/form/index.js

@@ -43,6 +43,7 @@ export const Form = React.createClass({
   getInitialState: function () {
     return {
       inputs: {},
+      dirty: {},
       errors: []
     };
   },
@@ -84,10 +85,16 @@ export const Form = React.createClass({
   },
 
   onChange: function (inputId, value) {
+    // update the internal key/value store, in addition to marking as dirty
     const inputState = Object.assign({}, this.state.inputs);
     set(inputState, [inputId, 'value'], value);
+
+    const markedDirty = Object.assign({}, this.state.dirty);
+    markedDirty[inputId] = true;
+
     this.setState(Object.assign({}, this.state, {
-      inputs: inputState
+      inputs: inputState,
+      dirty: markedDirty
     }));
   },
 
@@ -128,6 +135,13 @@ export const Form = React.createClass({
         delete inputState[inputId].error;
       }
 
+      // don't set a value for passwords that haven't changed,
+      // since these are garbled and encrypted.
+      const markedDirty = this.state.dirty[inputId];
+      if (!markedDirty && input.isPassword) {
+        return;
+      }
+
       // Ignore empty fields that aren't required
       // These may have input elements in the form, but
       // the API will fail if it's sent empty strings
@@ -196,6 +210,8 @@ export const Form = React.createClass({
             const mode = type === formTypes.textArea && form.mode || null;
             // subforms have fieldsets that define child form structure
             const fieldSet = type === formTypes.subform && form.fieldSet || null;
+            // text forms can be type=password
+            const textType = (type === formTypes.text && form.isPassword) ? 'password' : null;
             const elem = React.createElement(element, {
               id: inputId,
               label,
@@ -204,6 +220,7 @@ export const Form = React.createClass({
               mode,
               options,
               fieldSet,
+              type: textType,
               onChange: this.onChange
             });
             return <li className='form__item' key={inputId}>{elem}</li>;

app/scripts/components/form/schema.js

@@ -97,6 +97,7 @@ function textfield (config, property, validate) {
   config.type = formTypes.text;
   config.validate = validate;
   config.error = validate && get(errors, property, errors.required);
+  if (property === 'password') config.isPassword = true;
   return config;
 }
 

app/scripts/components/granules/granule.js

@@ -147,10 +147,9 @@ var GranuleOverview = React.createClass({
       ['Ingest', 'ingesting'],
       ['Processing', 'processing'],
       ['Pushed to CMR', 'cmr'],
-      ['Archiving', 'archiving']
+      ['Archiving', 'archiving'],
+      ['Complete', 'completed']
     ];
-    if (status === 'failed') statusList.push(['Failed', 'failed']);
-    else statusList.push(['Complete', 'completed']);
     const indicatorClass = 'progress-bar__indicator progress-bar__indicator--' + status;
     const statusBarClass = 'progress-bar__progress progress-bar__progress--' + status;
     return (

app/scripts/reducers/granules.js

@@ -42,6 +42,8 @@ import {
   OPTIONS_COLLECTIONNAME_ERROR
 } from '../actions';
 
+// manually filter out list items that have been deleted.
+// https://github.com/cumulus-nasa/cumulus-dashboard/issues/276
 function removeDeleted (list, deleted) {
   const filter = (item) => !(deleted[item.granuleId] && deleted[item.granuleId].status === 'success');
   return list.filter(filter);

ngap/bamboo/build_modules_package.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+cp -R /source /build
+(set -e && cd /build && npm install)
+tar -cf /artifacts/modules.tar -C /build node_modules
+chown "${RELEASE_UID}:${RELEASE_GID}" /artifacts/modules.tar

ngap/bamboo/build_release_package.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+
+cp -R /source /build
+(
+  set -e
+  cd /build
+  tar -xf modules.tar
+
+  npm run production
+
+  mkdir release
+  mkdir release/html
+  cp -R dist release/html/dashboard
+  cp ngap/deployment/Dockerfile release/Dockerfile
+  cp ngap/deployment/nginx.conf release/nginx.conf
+  cp ngap/deployment/Procfile release/Procfile
+  cp ngap/deployment/run_web.sh release/run_web.sh
+
+  tar -cf /artifacts/release.tar -C release .
+)
+
+chown "${RELEASE_UID}:${RELEASE_GID}" /artifacts/release.tar